From 9b6a9e838e1ec596d026b0ba923dd93130c433e1 Mon Sep 17 00:00:00 2001 From: Gabriele Quaresima Date: Mon, 7 Sep 2026 15:00:06 +0200 Subject: [PATCH 01/23] feat: manage backup retention in Klio instead of Kopia Replace Kopia's snapshot-expire retention with a Klio-managed policy manager that evaluates the backup catalog and deletes the expired backups itself. The only criterion for now is latest, which keeps the N most recent backups; it is configured per tier in the PluginConfiguration and must be at least 1. Because retention no longer runs through Kopia, the tier2 snapshot pins are gone. The invariant that a tier1 backup is never deleted before it reaches tier2 now lives in the retention selection, read from our own catalog, which keeps snapshot manifest IDs stable. The tier1 and tier2 policies travel to the server on the CloseBackup request, and the consumer applies them per tier after every backup. The klio retention CLI and the Kopia policy plumbing are removed. Assisted-by: Claude Signed-off-by: Gabriele Quaresima --- core/cmd/backup/run.go | 44 ++-- core/cmd/retention/doc.go | 21 -- core/cmd/retention/get.go | 107 --------- core/cmd/retention/set.go | 131 ----------- core/cmd/root.go | 2 - core/internal/client/klioclient/backup.go | 12 +- core/internal/client/klioclient/consts.go | 4 - core/internal/client/klioclient/interfaces.go | 18 +- .../client/klioclient/kopia/backup.go | 28 --- .../client/klioclient/kopia/compression.go} | 18 +- .../client/klioclient/kopia/multiconnect.go | 42 +--- .../klioclient/kopia/multiconnect_test.go | 60 +---- .../client/klioclient/kopia/retention.go | 52 ----- core/internal/cnpgi/backup.go | 77 +------ core/internal/cnpgi/retention.go | 80 ------- core/internal/consumer/backup.go | 119 +++++----- core/internal/consumer/backup_test.go | 73 ------- core/internal/consumer/maintenance.go | 69 ++++-- core/internal/consumer/maintenance_test.go | 24 +- core/internal/consumer/processbackup_test.go | 4 +- core/internal/grpc/klio_wal.pb.go | 18 +- core/internal/kopia/data.go | 36 --- core/internal/kopia/data_test.go | 12 - core/internal/kopia/pins.go | 79 ------- core/internal/kopia/policy.go | 104 --------- core/internal/kopia/write.go | 14 -- core/internal/queue/backup.go | 10 +- .../server/admin/delete_backup_test.go | 13 -- core/internal/server/walserver/backup.go | 36 ++- core/internal/server/walserver/backup_test.go | 47 ++++ core/pkg/config/client_validate.go | 12 + core/pkg/config/retention.go | 46 ++-- core/pkg/config/retention_test.go | 60 +++++ core/pkg/retention/retention.go | 88 ++++++++ core/pkg/retention/retention_test.go | 157 +++++++++++++ core/proto/klio_wal.proto | 3 + documentation/web/docs/developer/_protocol.md | 1 + documentation/web/docs/user/api/_klio_api.md | 10 +- documentation/web/docs/user/cli/klio.md | 1 - .../web/docs/user/cli/klio_retention.md | 38 ---- .../web/docs/user/cli/klio_retention_get.md | 40 ---- .../web/docs/user/cli/klio_retention_set.md | 46 ---- .../web/docs/user/concepts/architectures.md | 31 +-- .../web/docs/user/plugin_configuration.md | 43 +--- documentation/web/docs/user/upgrade_notes.md | 16 ++ .../v1alpha1/plugin_configuration_types.go | 30 +-- .../api/v1alpha1/zz_generated.deepcopy.go | 34 +-- .../klio.cnpg.io_pluginconfigurations.yaml | 72 ++---- .../chart/crds/pluginconfiguration-crd.yaml | 72 ++---- operator/internal/klioconfig/config.go | 7 +- operator/internal/klioconfig/config_test.go | 63 ++---- operator/pkg/config/retention.go | 46 ++-- .../e2e/pluginconfiguration_update_test.go | 2 +- operator/test/e2e/tier2_retention_test.go | 14 +- .../test/klio/features/tier2_retention.go | 206 ++++++------------ 55 files changed, 828 insertions(+), 1664 deletions(-) delete mode 100644 core/cmd/retention/doc.go delete mode 100644 core/cmd/retention/get.go delete mode 100644 core/cmd/retention/set.go rename core/{cmd/retention/root.go => internal/client/klioclient/kopia/compression.go} (65%) delete mode 100644 core/internal/client/klioclient/kopia/retention.go delete mode 100644 core/internal/cnpgi/retention.go delete mode 100644 core/internal/kopia/pins.go create mode 100644 core/internal/server/walserver/backup_test.go create mode 100644 core/pkg/config/retention_test.go create mode 100644 core/pkg/retention/retention.go create mode 100644 core/pkg/retention/retention_test.go delete mode 100644 documentation/web/docs/user/cli/klio_retention.md delete mode 100644 documentation/web/docs/user/cli/klio_retention_get.md delete mode 100644 documentation/web/docs/user/cli/klio_retention_set.md diff --git a/core/cmd/backup/run.go b/core/cmd/backup/run.go index 0b06522f..7971a138 100644 --- a/core/cmd/backup/run.go +++ b/core/cmd/backup/run.go @@ -126,13 +126,13 @@ func runBackup(cmd *cobra.Command, _ []string) error { backupfailure.RepositoryError.ExitCode) } - if err := backupExecutor.Upload(cmd.Context(), tier2); err != nil { + if err := backupExecutor.Upload(cmd.Context()); err != nil { return cli.NewCodedError( fmt.Errorf("while uploading data: %w", err), backupfailure.RepositoryError.ExitCode) } - metadata, err := backupExecutor.Close(cmd.Context(), tier2) + metadata, err := backupExecutor.Close(cmd.Context()) if err != nil { return cli.NewCodedError( fmt.Errorf("while closing the backup: %w", err), @@ -146,6 +146,16 @@ func runBackup(cmd *cobra.Command, _ []string) error { backupfailure.RepositoryError.ExitCode) } + tier1RetentionPolicy, err := configuration.Tier1RetentionPolicy.MarshalWire() + if err != nil { + contextLogger.Error(err, "Error while serializing the tier1 retention policy, skipping") + } + + tier2RetentionPolicy, err := configuration.Tier2RetentionPolicy.MarshalWire() + if err != nil { + contextLogger.Error(err, "Error while serializing the tier2 retention policy, skipping") + } + for { //nolint:gosec // postgres timeline is uint32 in practice, fits int32 timeline := int32(metadata.Timeline) @@ -158,7 +168,8 @@ func runBackup(cmd *cobra.Command, _ []string) error { EndWal: metadata.EndWAL, SegmentSize: metadata.SegmentSize, SendToTier2: tier2, - Tier2RetentionPolicy: marshalTier2RetentionPolicy(cmd.Context(), &configuration), + Tier1RetentionPolicy: tier1RetentionPolicy, + Tier2RetentionPolicy: tier2RetentionPolicy, Tier2CompressionPolicy: marshalTier2CompressionPolicy(cmd.Context(), &configuration), }) if err != nil { @@ -226,33 +237,6 @@ func toKopiaCompressionPolicy(p *config.CompressionPolicy) kopiaWrapper.Compress } } -// marshalTier2RetentionPolicy serializes the tier2 retention policy to the -// JSON representation expected by the WAL server. It returns an empty string -// when no policy is configured or serialization fails. -func marshalTier2RetentionPolicy(ctx context.Context, configuration *config.Data) string { - if configuration.Tier2RetentionPolicy == nil { - return "" - } - - policy := kopiaWrapper.RetentionPolicy{ - KeepLatest: configuration.Tier2RetentionPolicy.KeepLatest, - KeepHourly: configuration.Tier2RetentionPolicy.KeepHourly, - KeepDaily: configuration.Tier2RetentionPolicy.KeepDaily, - KeepWeekly: configuration.Tier2RetentionPolicy.KeepWeekly, - KeepMonthly: configuration.Tier2RetentionPolicy.KeepMonthly, - KeepAnnual: configuration.Tier2RetentionPolicy.KeepAnnual, - } - - content, err := json.Marshal(policy) - if err != nil { - log.FromContext(ctx).Error(err, "Error while serializing the tier2 retention policy, skipping") - - return "" - } - - return string(content) -} - // marshalTier2CompressionPolicy serializes the tier2 compression policy to the // JSON representation expected by the WAL server. It is always serialized, // even when unconfigured, so that removing the compression section resets the diff --git a/core/cmd/retention/doc.go b/core/cmd/retention/doc.go deleted file mode 100644 index ad63b05c..00000000 --- a/core/cmd/retention/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -// Package retention contains the implementation of the klio retention command -package retention diff --git a/core/cmd/retention/get.go b/core/cmd/retention/get.go deleted file mode 100644 index 0022ff22..00000000 --- a/core/cmd/retention/get.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package retention - -import ( - "encoding/json" - "fmt" - - "github.com/spf13/cobra" - "github.com/spf13/viper" - - "github.com/cloudnative-pg/klio/core/internal/cli" - "github.com/cloudnative-pg/klio/core/internal/client/klioclient/kopia" - kopiaWrapper "github.com/cloudnative-pg/klio/core/internal/kopia" - "github.com/cloudnative-pg/klio/core/pkg/config" -) - -// getCmd represents the retention get command -// -//nolint:gochecknoglobals -var getCmd = &cobra.Command{ - Use: "get", - Short: "Gets the currently applied retention policy", - RunE: func(cmd *cobra.Command, _ []string) error { - var configuration config.Data - - // IMPORTANT: this requires this program to be built with "-tags viper_bind_struct" - // when using environment variables - if err := viper.Unmarshal(&configuration); err != nil { - return fmt.Errorf("could not unmarshal configuration: %w", err) - } - - // Sets the default values, to be overridden by the user configuration - configuration.SetDefaults() - - if configuration.Client == (config.ClientConfig{}) { - return cli.ErrClientSectionIsRequired - } - if configuration.Client.Base == (config.BaseRepositoryClientConfig{}) { - return cli.ErrKopiaClientSectionIsRequired - } - - if err := configuration.Validate(); err != nil { - return fmt.Errorf("configuration validation error: %w", err) - } - - client, err := kopia.MultiConnect( - cmd.Context(), - &configuration.Client, - ) - if err != nil { - return fmt.Errorf("while connecting to the Klio server: %w %q", err, configuration.Client.Base.URL) - } - defer client.Close(cmd.Context()) - - effectivePolicy, err := client.GetRetentionPolicy( - cmd.Context(), - kopiaWrapper.Target{ - Hostname: client.GetHostname(), - Username: client.GetUsername(), - }, - ) - if err != nil { - return fmt.Errorf("while getting the current retention policy: %w", err) - } - - // Marshal metadata to JSON - jsonData, err := json.Marshal(effectivePolicy) - if err != nil { - return fmt.Errorf("failed to marshal metadata to JSON: %w", err) - } - - fmt.Println(string(jsonData)) //nolint:forbidigo - - return nil - }, -} - -//nolint:gochecknoinits -func init() { - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // runCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - - RetentionCmd.AddCommand(getCmd) -} diff --git a/core/cmd/retention/set.go b/core/cmd/retention/set.go deleted file mode 100644 index 00c41d6d..00000000 --- a/core/cmd/retention/set.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package retention - -import ( - "fmt" - "strconv" - - "github.com/spf13/cobra" - "github.com/spf13/viper" - - "github.com/cloudnative-pg/klio/core/internal/cli" - "github.com/cloudnative-pg/klio/core/internal/client/klioclient/kopia" - kopiaWrapper "github.com/cloudnative-pg/klio/core/internal/kopia" - "github.com/cloudnative-pg/klio/core/pkg/config" -) - -// setCmd represents the retention get command -// -//nolint:gochecknoglobals -var setCmd = &cobra.Command{ - Use: "set", - Short: "Sets the currently applied retention policy", - RunE: func(cmd *cobra.Command, _ []string) error { - var configuration config.Data - - // IMPORTANT: this requires this program to be built with "-tags viper_bind_struct" - // when using environment variables - if err := viper.Unmarshal(&configuration); err != nil { - return fmt.Errorf("could not unmarshal configuration: %w", err) - } - - // Sets the defaults values, to be overridden by the user configuration - configuration.SetDefaults() - - if configuration.Client == (config.ClientConfig{}) { - return cli.ErrClientSectionIsRequired - } - if configuration.Client.Base == (config.BaseRepositoryClientConfig{}) { - return cli.ErrKopiaClientSectionIsRequired - } - - if err := configuration.Validate(); err != nil { - return fmt.Errorf("configuration validation error: %w", err) - } - - client, err := kopia.MultiConnect( - cmd.Context(), - &configuration.Client, - ) - if err != nil { - return fmt.Errorf("while connecting to the Klio server: %w %q", err, configuration.Client.Base.URL) - } - defer client.Close(cmd.Context()) - - target := kopiaWrapper.Target{ - Hostname: client.GetHostname(), - Username: client.GetUsername(), - } - - effectivePolicy, err := client.GetRetentionPolicy(cmd.Context(), target) - if err != nil { - return fmt.Errorf("while getting the current retention policy: %w", err) - } - - getKeepValue := func(name string) *int { - f := cmd.Flags().Lookup(name) - if f == nil { - return nil - } - - if !f.Changed { - return nil - } - - value, err := strconv.Atoi(f.Value.String()) - if err != nil { - return nil - } - - return &value - } - - if effectivePolicy == nil { - effectivePolicy = &kopiaWrapper.RetentionPolicy{} - } - effectivePolicy.KeepLatest = getKeepValue("keep-latest") - effectivePolicy.KeepAnnual = getKeepValue("keep-annual") - effectivePolicy.KeepMonthly = getKeepValue("keep-monthly") - effectivePolicy.KeepWeekly = getKeepValue("keep-weekly") - effectivePolicy.KeepDaily = getKeepValue("keep-daily") - effectivePolicy.KeepHourly = getKeepValue("keep-hourly") - - if err := client.SetRetentionPolicy(cmd.Context(), target, *effectivePolicy); err != nil { - return fmt.Errorf("while setting the current retention policy: %w", err) - } - - return nil - }, -} - -//nolint:gochecknoinits -func init() { - // The following flags are really misleading. - // We should find a way to better document them. - setCmd.Flags().Int("keep-latest", 0, "Number of most recent latest backup kept") - setCmd.Flags().Int("keep-annual", 0, "Number of most recent annual backup kept") - setCmd.Flags().Int("keep-monthly", 0, "Number of most recent monthly backup kept") - setCmd.Flags().Int("keep-weekly", 0, "Number of most recent weekly backup kept") - setCmd.Flags().Int("keep-daily", 0, "Number of most recent daily backup kept") - setCmd.Flags().Int("keep-hourly", 0, "Number of most recent hourly backup kept") - - RetentionCmd.AddCommand(setCmd) -} diff --git a/core/cmd/root.go b/core/cmd/root.go index 15af469c..da337ba0 100644 --- a/core/cmd/root.go +++ b/core/cmd/root.go @@ -33,7 +33,6 @@ import ( "github.com/cloudnative-pg/klio/core/cmd/admin" "github.com/cloudnative-pg/klio/core/cmd/backup" "github.com/cloudnative-pg/klio/core/cmd/cnpgi" - "github.com/cloudnative-pg/klio/core/cmd/retention" "github.com/cloudnative-pg/klio/core/cmd/server" "github.com/cloudnative-pg/klio/core/cmd/walplayer" @@ -123,7 +122,6 @@ func init() { rootCmd.AddCommand(backup.BackupCmd) rootCmd.AddCommand(walplayer.WalPlayerCmd) rootCmd.AddCommand(cnpgi.CnpgiCmd) - rootCmd.AddCommand(retention.RetentionCmd) rootCmd.AddCommand(admin.AdminCmd) } diff --git a/core/internal/client/klioclient/backup.go b/core/internal/client/klioclient/backup.go index 2a6d544e..77a6f1a8 100644 --- a/core/internal/client/klioclient/backup.go +++ b/core/internal/client/klioclient/backup.go @@ -126,7 +126,7 @@ func (b *BackupExecutor) Start(ctx context.Context, opts BackupOptions) error { } // Upload starts the uploading process. -func (b *BackupExecutor) Upload(ctx context.Context, pinned bool) error { +func (b *BackupExecutor) Upload(ctx context.Context) error { contextLogger := log.FromContext(ctx) for i, tbl := range b.tablespaces { @@ -136,19 +136,19 @@ func (b *BackupExecutor) Upload(ctx context.Context, pinned bool) error { "current", i+1, "total", len(b.tablespaces), ) - if err := b.uploader.UploadTablespace(ctx, b.name, tbl, pinned); err != nil { + if err := b.uploader.UploadTablespace(ctx, b.name, tbl); err != nil { return err //nolint:wrapcheck } } contextLogger.Info("Uploading PGDATA") - if err := b.uploader.UploadPgData(ctx, b.name, b.pgData, pinned); err != nil { + if err := b.uploader.UploadPgData(ctx, b.name, b.pgData); err != nil { return err //nolint:wrapcheck } contextLogger.Info("Uploading control file") controlDataFileName := path.Join(b.pgData, controlDataPath) - if err := b.uploader.UploadControlFile(ctx, b.name, controlDataFileName, pinned); err != nil { + if err := b.uploader.UploadControlFile(ctx, b.name, controlDataFileName); err != nil { return err //nolint:wrapcheck } @@ -156,7 +156,7 @@ func (b *BackupExecutor) Upload(ctx context.Context, pinned bool) error { } // Close finishes a backup. -func (b *BackupExecutor) Close(ctx context.Context, pinned bool) (*BackupMetadata, error) { +func (b *BackupExecutor) Close(ctx context.Context) (*BackupMetadata, error) { contextLogger := log.FromContext(ctx) var tli int @@ -217,7 +217,7 @@ func (b *BackupExecutor) Close(ctx context.Context, pinned bool) (*BackupMetadat } contextLogger.Info("Uploading backup metadata") - if err := b.uploader.UploadBackupMetadata(ctx, b.name, metadata, pinned); err != nil { + if err := b.uploader.UploadBackupMetadata(ctx, b.name, metadata); err != nil { return nil, fmt.Errorf("while uploading backup metadata: %w", err) } diff --git a/core/internal/client/klioclient/consts.go b/core/internal/client/klioclient/consts.go index 1bd52d08..20f8170a 100644 --- a/core/internal/client/klioclient/consts.go +++ b/core/internal/client/klioclient/consts.go @@ -30,7 +30,3 @@ const BackupContentTagName = "klio.io/content" // TablespaceNameTagName is the name of the tag containing the // name of the tablespace. const TablespaceNameTagName = "klio.io/tablespaceName" - -// Tier2Pin is the name of the pin indicating that this -// snapshot should not be deleted until it is uploaded to tier2. -const Tier2Pin = "klio.io/tier2" diff --git a/core/internal/client/klioclient/interfaces.go b/core/internal/client/klioclient/interfaces.go index 26514056..c81009f1 100644 --- a/core/internal/client/klioclient/interfaces.go +++ b/core/internal/client/klioclient/interfaces.go @@ -29,16 +29,16 @@ import ( type BackupExecutorSupport interface { // UploadTablespace uploads the tablespace with the passed layout to // the backup store. - UploadTablespace(ctx context.Context, backupName string, tbl TablespaceLayout, pinned bool) error + UploadTablespace(ctx context.Context, backupName string, tbl TablespaceLayout) error // UploadPgData uploads the PGData to the backup store. - UploadPgData(ctx context.Context, backupName string, pgData string, pinned bool) error + UploadPgData(ctx context.Context, backupName string, pgData string) error // UploadControlFile uploads the control file to the backup store. - UploadControlFile(ctx context.Context, backupName string, controlDataFileName string, pinned bool) error + UploadControlFile(ctx context.Context, backupName string, controlDataFileName string) error // UploadBackupMetadata is called to upload the control file and to mark a backup successfully done. - UploadBackupMetadata(ctx context.Context, backupName string, metadata *BackupMetadata, pinned bool) error + UploadBackupMetadata(ctx context.Context, backupName string, metadata *BackupMetadata) error } // BackupRestoreSupport contains the functions needed to restore a backup. @@ -83,19 +83,9 @@ type Client interface { // This is read from the client certificate. GetHostname() string - // SetRetentionPolicy sets the retention policy for backups of this cluster. - SetRetentionPolicy(ctx context.Context, t kopia.Target, p kopia.RetentionPolicy) error - // SetCompressionPolicy sets the compression policy for backups of this cluster. SetCompressionPolicy(ctx context.Context, t kopia.Target, policy kopia.CompressionPolicy) error - // GetRetentionPolicy gets the currently applied retention policy for this cluster. - GetRetentionPolicy(ctx context.Context, t kopia.Target) (*kopia.RetentionPolicy, error) - - // ApplyRetentionPolicy applies the retention policy for this cluster, deleting any - // snapshots that are no longer needed. - ApplyRetentionPolicy(ctx context.Context, t kopia.Target) error - // Close closes the underlying connection. Close(ctx context.Context) } diff --git a/core/internal/client/klioclient/kopia/backup.go b/core/internal/client/klioclient/kopia/backup.go index 462e2fed..121d114f 100644 --- a/core/internal/client/klioclient/kopia/backup.go +++ b/core/internal/client/klioclient/kopia/backup.go @@ -38,7 +38,6 @@ func (s *Connection) UploadTablespace( ctx context.Context, backupName string, tbl klioclient.TablespaceLayout, - pinned bool, ) error { tags := map[string]string{ klioclient.BackupContentTagName: "tablespace", @@ -46,16 +45,10 @@ func (s *Connection) UploadTablespace( klioclient.BackupNameTagName: backupName, } - var pins []string - if pinned { - pins = []string{klioclient.Tier2Pin} - } - err := s.kopia.SnapshotDirectory(ctx, kopia.SnapshotDirectoryOptions{ Directory: tbl.Path, Tags: tags, Description: fmt.Sprintf("tablespace %s (%v)", tbl.Name, tbl.Oid), - Pins: pins, }) if err != nil { return fmt.Errorf("unable to snapshot directory: %w", err) @@ -68,7 +61,6 @@ func (s *Connection) UploadTablespace( func (s *Connection) UploadPgData( ctx context.Context, backupName, pgData string, - pinned bool, ) error { contextLogger := log.FromContext(ctx) @@ -89,16 +81,10 @@ func (s *Connection) UploadPgData( klioclient.BackupNameTagName: backupName, } - var pins []string - if pinned { - pins = []string{klioclient.Tier2Pin} - } - err := s.kopia.SnapshotDirectory(ctx, kopia.SnapshotDirectoryOptions{ Directory: pgData, Tags: tags, Description: "pgdata", - Pins: pins, }) if err != nil { return err @@ -116,23 +102,16 @@ func (s *Connection) UploadPgData( func (s *Connection) UploadControlFile( ctx context.Context, backupName, controlDataFileName string, - pinned bool, ) error { tags := map[string]string{ klioclient.BackupNameTagName: backupName, klioclient.BackupContentTagName: "controldata", } - var pins []string - if pinned { - pins = []string{klioclient.Tier2Pin} - } - err := s.kopia.SnapshotDirectory(ctx, kopia.SnapshotDirectoryOptions{ Directory: controlDataFileName, Tags: tags, Description: "control data file", - Pins: pins, }) if err != nil { return fmt.Errorf("while snapshotting control data file: %w", err) @@ -146,7 +125,6 @@ func (s *Connection) UploadBackupMetadata( ctx context.Context, backupName string, data *klioclient.BackupMetadata, - pinned bool, ) error { metadataContent, err := json.Marshal(data) if err != nil { @@ -160,18 +138,12 @@ func (s *Connection) UploadBackupMetadata( fakeMetadataDirectory := strings.TrimSuffix(data.PgData, "/") + "_meta" - var pins []string - if pinned { - pins = []string{klioclient.Tier2Pin} - } - opts := kopia.SnapshotFileContentOptions{ Content: metadataContent, FileName: "metadata.json", DirectoryName: fakeMetadataDirectory, Description: "metadata for " + backupName, Tags: tags, - Pins: pins, } if err := s.kopia.SnapshotFileContent(ctx, opts); err != nil { diff --git a/core/cmd/retention/root.go b/core/internal/client/klioclient/kopia/compression.go similarity index 65% rename from core/cmd/retention/root.go rename to core/internal/client/klioclient/kopia/compression.go index db1572cb..c69ed186 100644 --- a/core/cmd/retention/root.go +++ b/core/internal/client/klioclient/kopia/compression.go @@ -17,19 +17,15 @@ limitations under the License. SPDX-License-Identifier: Apache-2.0 */ -package retention +package kopia import ( - "github.com/spf13/cobra" + "context" + + "github.com/cloudnative-pg/klio/core/internal/kopia" ) -// RetentionCmd the `klio backup` command -// -//nolint:gochecknoglobals -var RetentionCmd = &cobra.Command{ - Use: "retention", - Short: "Manage the retention policy", - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, +// SetCompressionPolicy sets the compression policy for backups of this cluster. +func (s *Connection) SetCompressionPolicy(ctx context.Context, t kopia.Target, policy kopia.CompressionPolicy) error { + return s.kopia.SetKopiaCompressionPolicy(ctx, t, policy) } diff --git a/core/internal/client/klioclient/kopia/multiconnect.go b/core/internal/client/klioclient/kopia/multiconnect.go index 86d73eca..5deda3ea 100644 --- a/core/internal/client/klioclient/kopia/multiconnect.go +++ b/core/internal/client/klioclient/kopia/multiconnect.go @@ -111,19 +111,6 @@ func (s *MultiConnection) DeleteBackup(ctx context.Context, hostname string, nam return s.Tier1.DeleteBackup(ctx, hostname, name) } -// SetRetentionPolicy implements the Client interface. -func (s *MultiConnection) SetRetentionPolicy( - ctx context.Context, - t kopia.Target, - p kopia.RetentionPolicy, -) error { - if s.Tier1 == nil { - return ErrUnsupportedWriteOperation - } - - return s.Tier1.SetRetentionPolicy(ctx, t, p) -} - // SetCompressionPolicy implements the Client interface. func (s *MultiConnection) SetCompressionPolicy( ctx context.Context, @@ -137,14 +124,6 @@ func (s *MultiConnection) SetCompressionPolicy( return s.Tier1.SetCompressionPolicy(ctx, t, policy) } -// GetRetentionPolicy implements the Client interface. -func (s *MultiConnection) GetRetentionPolicy( - ctx context.Context, - t kopia.Target, -) (*kopia.RetentionPolicy, error) { - return s.getReadClient().GetRetentionPolicy(ctx, t) -} - // GetMetadata implements the BackupRestoreSupport interface. func (s *MultiConnection) GetMetadata( ctx context.Context, @@ -174,15 +153,6 @@ func (s *MultiConnection) GetMetadata( return markTier2(meta), nil } -// ApplyRetentionPolicy implements the Client interface. -func (s *MultiConnection) ApplyRetentionPolicy(ctx context.Context, t kopia.Target) error { - if s.Tier1 == nil { - return ErrUnsupportedWriteOperation - } - - return s.Tier1.ApplyRetentionPolicy(ctx, t) -} - // ListBackups implements the BackupRestoreSupport interface. // //nolint:cyclop @@ -277,13 +247,12 @@ func (s *MultiConnection) UploadTablespace( ctx context.Context, backupName string, tbl klioclient.TablespaceLayout, - pinned bool, ) error { if s.Tier1 == nil { return ErrUnsupportedWriteOperation } - return s.Tier1.UploadTablespace(ctx, backupName, tbl, pinned) + return s.Tier1.UploadTablespace(ctx, backupName, tbl) } // UploadPgData implements the BackupExecutorSupport interface. @@ -291,13 +260,12 @@ func (s *MultiConnection) UploadPgData( ctx context.Context, backupName string, pgData string, - pinned bool, ) error { if s.Tier1 == nil { return ErrUnsupportedWriteOperation } - return s.Tier1.UploadPgData(ctx, backupName, pgData, pinned) + return s.Tier1.UploadPgData(ctx, backupName, pgData) } // UploadControlFile implements the BackupExecutorSupport interface. @@ -305,13 +273,12 @@ func (s *MultiConnection) UploadControlFile( ctx context.Context, backupName string, controlDataFileName string, - pinned bool, ) error { if s.Tier1 == nil { return ErrUnsupportedWriteOperation } - return s.Tier1.UploadControlFile(ctx, backupName, controlDataFileName, pinned) + return s.Tier1.UploadControlFile(ctx, backupName, controlDataFileName) } // UploadBackupMetadata implements the BackupExecutorSupport interface. @@ -319,13 +286,12 @@ func (s *MultiConnection) UploadBackupMetadata( ctx context.Context, backupName string, metadata *klioclient.BackupMetadata, - pinned bool, ) error { if s.Tier1 == nil { return ErrUnsupportedWriteOperation } - return s.Tier1.UploadBackupMetadata(ctx, backupName, metadata, pinned) + return s.Tier1.UploadBackupMetadata(ctx, backupName, metadata) } func (s *MultiConnection) getClientFromMetadata(meta *klioclient.BackupMetadata) klioclient.Client { diff --git a/core/internal/client/klioclient/kopia/multiconnect_test.go b/core/internal/client/klioclient/kopia/multiconnect_test.go index ba28f8b8..f1b6c71f 100644 --- a/core/internal/client/klioclient/kopia/multiconnect_test.go +++ b/core/internal/client/klioclient/kopia/multiconnect_test.go @@ -28,7 +28,6 @@ import ( "github.com/stretchr/testify/require" "github.com/cloudnative-pg/klio/core/internal/client/klioclient" - "github.com/cloudnative-pg/klio/core/internal/kopia" ) // MockKlioClient is a mock implementation of klioclient.Client. @@ -41,20 +40,15 @@ type MockKlioClient struct { GetMetadataFunc func(ctx context.Context, hostname string, name string) ( *klioclient.BackupMetadata, error) - DeleteBackupFunc func(ctx context.Context, hostname string, name string) error - SetRetentionPolicyFunc func(ctx context.Context, t kopia.Target, - p kopia.RetentionPolicy) error - GetRetentionPolicyFunc func(ctx context.Context, t kopia.Target) ( - *kopia.RetentionPolicy, error) - ApplyRetentionPolicyFunc func(ctx context.Context, t kopia.Target) error - UploadTablespaceFunc func(ctx context.Context, backupName string, - tbl klioclient.TablespaceLayout, pinned bool) error + DeleteBackupFunc func(ctx context.Context, hostname string, name string) error + UploadTablespaceFunc func(ctx context.Context, backupName string, + tbl klioclient.TablespaceLayout) error UploadPgDataFunc func(ctx context.Context, backupName string, - pgData string, pinned bool) error + pgData string) error UploadControlFileFunc func(ctx context.Context, backupName string, - controlDataFileName string, pinned bool) error + controlDataFileName string) error UploadBackupMetadataFunc func(ctx context.Context, backupName string, - metadata *klioclient.BackupMetadata, pinned bool) error + metadata *klioclient.BackupMetadata) error RestoreTablespaceFunc func(ctx context.Context, metadata *klioclient.BackupMetadata, tbl klioclient.TablespaceLayout, destinationDirectory string) error @@ -96,58 +90,29 @@ func (m *MockKlioClient) DeleteBackup(ctx context.Context, hostname string, name return nil } -func (m *MockKlioClient) SetRetentionPolicy( - ctx context.Context, t kopia.Target, p kopia.RetentionPolicy, -) error { - if m.SetRetentionPolicyFunc != nil { - return m.SetRetentionPolicyFunc(ctx, t, p) - } - - return nil -} - -func (m *MockKlioClient) GetRetentionPolicy( - ctx context.Context, t kopia.Target, -) (*kopia.RetentionPolicy, error) { - if m.GetRetentionPolicyFunc != nil { - return m.GetRetentionPolicyFunc(ctx, t) - } - - return nil, nil -} - -func (m *MockKlioClient) ApplyRetentionPolicy(ctx context.Context, t kopia.Target) error { - if m.ApplyRetentionPolicyFunc != nil { - return m.ApplyRetentionPolicyFunc(ctx, t) - } - - return nil -} - func (m *MockKlioClient) UploadTablespace( ctx context.Context, backupName string, tbl klioclient.TablespaceLayout, - pinned bool, ) error { if m.UploadTablespaceFunc != nil { - return m.UploadTablespaceFunc(ctx, backupName, tbl, pinned) + return m.UploadTablespaceFunc(ctx, backupName, tbl) } return nil } -func (m *MockKlioClient) UploadPgData(ctx context.Context, backupName string, pgData string, pinned bool) error { +func (m *MockKlioClient) UploadPgData(ctx context.Context, backupName string, pgData string) error { if m.UploadPgDataFunc != nil { - return m.UploadPgDataFunc(ctx, backupName, pgData, pinned) + return m.UploadPgDataFunc(ctx, backupName, pgData) } return nil } func (m *MockKlioClient) UploadControlFile( - ctx context.Context, backupName string, controlDataFileName string, pinned bool, + ctx context.Context, backupName string, controlDataFileName string, ) error { if m.UploadControlFileFunc != nil { - return m.UploadControlFileFunc(ctx, backupName, controlDataFileName, pinned) + return m.UploadControlFileFunc(ctx, backupName, controlDataFileName) } return nil @@ -155,10 +120,9 @@ func (m *MockKlioClient) UploadControlFile( func (m *MockKlioClient) UploadBackupMetadata( ctx context.Context, backupName string, metadata *klioclient.BackupMetadata, - pinned bool, ) error { if m.UploadBackupMetadataFunc != nil { - return m.UploadBackupMetadataFunc(ctx, backupName, metadata, pinned) + return m.UploadBackupMetadataFunc(ctx, backupName, metadata) } return nil diff --git a/core/internal/client/klioclient/kopia/retention.go b/core/internal/client/klioclient/kopia/retention.go deleted file mode 100644 index e4428f55..00000000 --- a/core/internal/client/klioclient/kopia/retention.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package kopia - -import ( - "context" - - "github.com/cloudnative-pg/klio/core/internal/kopia" -) - -// SetRetentionPolicy sets the retention policy for backups of this cluster. -func (s *Connection) SetRetentionPolicy(ctx context.Context, t kopia.Target, p kopia.RetentionPolicy) error { - return s.kopia.SetKopiaPolicy(ctx, t, &p) -} - -// SetCompressionPolicy sets the compression policy for backups of this cluster. -func (s *Connection) SetCompressionPolicy(ctx context.Context, t kopia.Target, policy kopia.CompressionPolicy) error { - return s.kopia.SetKopiaCompressionPolicy(ctx, t, policy) -} - -// GetRetentionPolicy gets the currently applied retention policy for this cluster. -func (s *Connection) GetRetentionPolicy(ctx context.Context, t kopia.Target) (*kopia.RetentionPolicy, error) { - policy, err := s.kopia.GetCurrentKopiaPolicy(ctx, t) - if err != nil { - return nil, err - } - - return &policy.RetentionPolicy, nil -} - -// ApplyRetentionPolicy applies the retention policy for this cluster, deleting any -// snapshots that are no longer needed. -func (s *Connection) ApplyRetentionPolicy(ctx context.Context, t kopia.Target) error { - return s.kopia.ApplyKopiaPolicy(ctx, t) -} diff --git a/core/internal/cnpgi/backup.go b/core/internal/cnpgi/backup.go index 90688b32..386bca1f 100644 --- a/core/internal/cnpgi/backup.go +++ b/core/internal/cnpgi/backup.go @@ -27,7 +27,6 @@ import ( "fmt" "os" "os/exec" - "strconv" "strings" "time" @@ -79,25 +78,13 @@ func (b backupServiceImplementation) Backup( ctx, span := tracer.Start(ctx, opentelemetry.BackupSpan) defer span.End() - // Step 1: get and apply the retention policies var cluster cnpgv1.Cluster if err := json.Unmarshal(request.GetClusterDefinition(), &cluster); err != nil { return nil, fmt.Errorf("failed to unmarshal cluster definition: %w", err) } - r, err := extractTier1RetentionFromConfiguration() - if err != nil { - return nil, fmt.Errorf("failed to extract retention policy from configuration: %w", err) - } - - if err = b.setRetentionPolicy(ctx, r); err != nil { - // Yes this is intentional. If we don't set the retention policies from - // the configuration file, it is not a major issue. We can continue with the backup. - // The eventual error will be logged into the setRetentionPolicy function - log.Error(err, "failed to set retention policy") - } - - // Step 2: starting the backup + // The backup consumer applies retention server-side from the policy carried + // on the CloseBackup request, so the plugin only starts the backup. backupName := fmt.Sprintf("backup-%v", pgTime.ToCompactISO8601(time.Now())) isPrimary := b.InstanceName == cluster.Status.CurrentPrimary @@ -328,66 +315,6 @@ func (b backupServiceImplementation) runVerify(ctx context.Context, backupName s return false, nil } -//nolint:cyclop -func (b backupServiceImplementation) setRetentionPolicy(ctx context.Context, r *Retention) error { - contextLogger := log.FromContext(ctx) - - if r.IsEmpty() { - contextLogger.Info("Skipping retention policy creation") - return nil - } - - klioPath, err := os.Executable() - if err != nil { - return fmt.Errorf("failed to determine klio path: %w", err) - } - - klioArgs := []string{ - "retention", "set", "--config", backupRepositoryConfigPath, - } - if r.KeepAnnual != nil { - klioArgs = append(klioArgs, "--keep-annual", strconv.Itoa(*r.KeepAnnual)) - } - if r.KeepDaily != nil { - klioArgs = append(klioArgs, "--keep-daily", strconv.Itoa(*r.KeepDaily)) - } - if r.KeepHourly != nil { - klioArgs = append(klioArgs, "--keep-hourly", strconv.Itoa(*r.KeepHourly)) - } - if r.KeepLatest != nil { - klioArgs = append(klioArgs, "--keep-latest", strconv.Itoa(*r.KeepLatest)) - } - if r.KeepWeekly != nil { - klioArgs = append(klioArgs, "--keep-weekly", strconv.Itoa(*r.KeepWeekly)) - } - if r.KeepMonthly != nil { - klioArgs = append(klioArgs, "--keep-monthly", strconv.Itoa(*r.KeepMonthly)) - } - - contextLogger.Info("Executing klio retention set", "args", klioArgs) - //nolint:gosec - cmd := exec.CommandContext(ctx, klioPath, klioArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to execute 'klio retention set' command: %w", err) - } - - contextLogger.Info("Executing klio retention get") - //nolint:gosec - cmd = exec.CommandContext(ctx, klioPath, "retention", "get", "--config", backupRepositoryConfigPath) - var stdout bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to execute 'klio retention get' command: %w", err) - } - - contextLogger.Info("Effective retention policy", "effectivePolicy", stdout.String()) - - return nil -} - // filterOTelEnv returns a copy of env with all OTEL_ variables removed. // This prevents subprocesses from inheriting OpenTelemetry configuration // (e.g. OTEL_METRICS_EXPORTER=console) that could write to stdout and diff --git a/core/internal/cnpgi/retention.go b/core/internal/cnpgi/retention.go deleted file mode 100644 index e2834c3c..00000000 --- a/core/internal/cnpgi/retention.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package cnpgi - -import ( - "fmt" - - "github.com/spf13/afero" - "sigs.k8s.io/yaml" - - "github.com/cloudnative-pg/klio/core/pkg/config" -) - -// Retention contains the retention policy configuration. -type Retention struct { - KeepLatest *int - KeepAnnual *int - KeepMonthly *int - KeepWeekly *int - KeepDaily *int - KeepHourly *int -} - -// IsEmpty checks if the retention configuration is empty or not. -func (r *Retention) IsEmpty() bool { - if r == nil { - return true - } - - emptyRetention := Retention{} - - return *r == emptyRetention -} - -// extractTier1RetentionFromConfiguration reads retention policy settings. -func extractTier1RetentionFromConfiguration() (*Retention, error) { - osFS := afero.NewOsFs() - f, err := afero.ReadFile(osFS, backupRepositoryConfigPath) - if err != nil { - return nil, fmt.Errorf("failed to read backup repository config file %q: %w", backupRepositoryConfigPath, err) - } - - var configData config.Data - err = yaml.Unmarshal(f, &configData) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal backup repository config file %q: %w", backupRepositoryConfigPath, err) - } - - conf := Retention{} - - if configData.Tier1RetentionPolicy == nil { - return &conf, nil - } - - conf.KeepLatest = configData.Tier1RetentionPolicy.KeepLatest - conf.KeepAnnual = configData.Tier1RetentionPolicy.KeepAnnual - conf.KeepMonthly = configData.Tier1RetentionPolicy.KeepMonthly - conf.KeepWeekly = configData.Tier1RetentionPolicy.KeepWeekly - conf.KeepDaily = configData.Tier1RetentionPolicy.KeepDaily - conf.KeepHourly = configData.Tier1RetentionPolicy.KeepHourly - - return &conf, nil -} diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index 9ba0266d..fefe4a13 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -35,6 +35,7 @@ import ( "github.com/cloudnative-pg/klio/core/internal/opentelemetry" "github.com/cloudnative-pg/klio/core/internal/queue" "github.com/cloudnative-pg/klio/core/internal/repository" + "github.com/cloudnative-pg/klio/core/pkg/retention" ) // errTier2NotConfigured is returned when a backup requests a tier2 relay but @@ -49,8 +50,8 @@ type backupSteps interface { listManifests(ctx context.Context, clusterName string) ([]kopia.Manifest, error) verifyTier1(ctx context.Context, clusterName string) error relayTier2(ctx context.Context, task *queue.BackupTask, entries []kopia.Manifest) error - maintainTier2(ctx context.Context, task *queue.BackupTask, entries []kopia.Manifest) error - maintainTier1(ctx context.Context, clusterName string, entries []kopia.Manifest) error + maintainTier2(ctx context.Context, task *queue.BackupTask) error + maintainTier1(ctx context.Context, task *queue.BackupTask) error } // Backup represents a Backup consumer. @@ -210,7 +211,7 @@ func (d *Backup) relayAndMaintain(ctx context.Context, task *queue.BackupTask, e // tier2 maintenance (retention + WAL cleanup) records its own per-tier // metric; a tier2 base-retention failure is fatal (the task is retried) // while WAL cleanup is best-effort. - if err := d.steps.maintainTier2(ctx, task, entries); err != nil { + if err := d.steps.maintainTier2(ctx, task); err != nil { return err } } @@ -220,7 +221,7 @@ func (d *Backup) relayAndMaintain(ctx context.Context, task *queue.BackupTask, e // misconfigured one above. It records its own per-tier metric (the only // signal of a tier1 maintenance failure, which is otherwise best-effort); // we log but don't fail the task on its error. - if err := d.steps.maintainTier1(ctx, task.ClusterName, entries); err != nil { + if err := d.steps.maintainTier1(ctx, task); err != nil { contextLogger.Error(err, "Error while applying tier1 maintenance, skipping") } @@ -270,53 +271,24 @@ func (d *Backup) relayTier2(ctx context.Context, task *queue.BackupTask, entries return d.verifyTier2Backups(ctx, task.ClusterName) } -// maintainTier2 enforces tier2 retention (base-snapshot policy and WAL -// cleanup) after a successful relay, and records the tier2 maintenance metric. -// A base-retention failure (policy set/apply) is fatal so the task is retried; -// unpin, server refresh and WAL cleanup are best-effort. The ordering matches -// the previous inline flow: WAL retention runs after the server refresh so it -// lists the post-retention backups. -func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask, entries []kopia.Manifest) error { +// maintainTier2 enforces tier2 retention (Klio-managed base-backup deletion and +// WAL cleanup) after a successful relay, and records the tier2 maintenance +// metric. A base-retention failure is fatal so the task is retried; server +// refresh and WAL cleanup are best-effort. WAL retention runs after the server +// refresh so it lists the post-retention backups. +func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask) error { contextLogger := log.FromContext(ctx) contextLogger.Info("Applying tier2 maintenance", "cluster", task.ClusterName) - target := kopia.Target{ - Username: entries[0].Source.UserName, - Hostname: task.ClusterName, - } - - if task.Tier2RetentionPolicy != nil { - if err := d.tier2Kopia.SetKopiaPolicy(ctx, target, task.Tier2RetentionPolicy); err != nil { - recordMaintenance(ctx, task.ClusterName, opentelemetry.Tier2, err) - - return err - } - } - - if err := d.tier2Kopia.ApplyKopiaPolicy(ctx, target); err != nil { + if err := d.applyRetention(ctx, d.tier2Client, task.ClusterName, task.Tier2RetentionPolicy, nil); err != nil { recordMaintenance(ctx, task.ClusterName, opentelemetry.Tier2, err) return err } - // Unpin the pinned snapshots (best-effort: the backup is already on tier2; - // they will be unpinned when migrating the next backup). - if pinnedSnapshots := getPinnedSnapshots(entries); len(pinnedSnapshots) > 0 { - if err := d.tier1Kopia.PinSnapshots(ctx, kopia.PinSnapshotOpts{ - IDs: pinnedSnapshots, - RemovePins: []string{klioclient.Tier2Pin}, - }); err != nil { - contextLogger.Error(err, "Error while unpinning snapshots") - } - } - - // Refresh the server cache so it reflects the post-retention manifest + // Refresh the tier2 server cache so it reflects the post-retention manifest // list before WAL retention lists the surviving backups (best-effort). - contextLogger.Info("Refreshing tier1 and tier 2 Kopia server cache to reflect post-retention manifest list") - if err := d.refreshTier1KopiaServer(ctx); err != nil { - contextLogger.Error(err, "Error while refreshing tier1 Kopia server, continuing") - } - + contextLogger.Info("Refreshing tier2 Kopia server cache to reflect post-retention manifest list") if err := d.refreshTier2KopiaServer(ctx); err != nil { contextLogger.Error(err, "Error while refreshing Kopia server cache, skipping") } @@ -335,13 +307,50 @@ func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask, entr return nil } -func (d *Backup) refreshTier1KopiaServer(ctx context.Context) error { - return d.tier1Kopia.RefreshServer(ctx, kopia.RefreshServerOptions{ - ServerControlUser: d.opts.RunID, - ServerControlPassword: d.opts.RunSecret, - ServerCertFingerprint: d.opts.Tier1ServerCertificateFingerprint, - Address: d.opts.Tier1ServerAddress, - }) +// applyRetention deletes the base backups of a cluster that fall outside the +// given policy, evaluated against Klio's own catalog. The optional keep +// predicate protects a backup from deletion even when the policy expired it +// (tier1 uses it to never delete a backup that is not yet on tier2). A zero +// policy deletes nothing. +func (d *Backup) applyRetention( + ctx context.Context, + client *klioclientkopia.Connection, + clusterName string, + policy retention.Policy, + keep func(name string) bool, +) error { + contextLogger := log.FromContext(ctx) + + backups, err := client.ListBackups(ctx, clusterName) + if err != nil { + return fmt.Errorf("while listing backups for cluster %q: %w", clusterName, err) + } + + catalog := make([]retention.Backup, len(backups)) + for i := range backups { + catalog[i] = retention.Backup{ + Name: backups[i].Name, + StartedAt: backups[i].StartedAt, + StoppedAt: backups[i].StoppedAt, + } + } + + var errs error + for _, expired := range retention.Evaluate(catalog, policy) { + if keep != nil && keep(expired.Name) { + contextLogger.Info("Retention expired a backup that is not deletable yet, keeping it", + "cluster", clusterName, "backup", expired.Name) + + continue + } + + contextLogger.Info("Deleting expired backup", "cluster", clusterName, "backup", expired.Name) + if err := client.DeleteBackup(ctx, clusterName, expired.Name); err != nil { + errs = errors.Join(errs, fmt.Errorf("while deleting expired backup %q: %w", expired.Name, err)) + } + } + + return errs } func (d *Backup) listManifests(ctx context.Context, cluster string) ([]kopia.Manifest, error) { @@ -372,18 +381,6 @@ func manifestListToDescriptors(entries []kopia.Manifest) []string { return result.ToSortedList() } -func getPinnedSnapshots(manifests []kopia.Manifest) []string { - result := stringset.New() - - for i := range manifests { - if len(manifests[i].Pins) > 0 && manifests[i].RootEntry != nil && manifests[i].RootEntry.ObjID != "" { - result.Put(manifests[i].RootEntry.ObjID) - } - } - - return result.ToSortedList() -} - // refreshTier2KopiaServer makes sure the tier 2 kopia server // has downloaded the latest manifests from the object store. func (d *Backup) refreshTier2KopiaServer(ctx context.Context) error { diff --git a/core/internal/consumer/backup_test.go b/core/internal/consumer/backup_test.go index 88fec7f9..7c736ac7 100644 --- a/core/internal/consumer/backup_test.go +++ b/core/internal/consumer/backup_test.go @@ -79,79 +79,6 @@ func TestManifestListToDescriptors(t *testing.T) { } } -func TestGetPinnedSnapshots(t *testing.T) { - cases := []struct { - name string - manifests []kopia.Manifest - expected []string - }{ - { - name: "Empty input returns empty slice", - manifests: []kopia.Manifest{}, - expected: []string{}, - }, - { - name: "No pinned snapshots returns empty slice", - manifests: []kopia.Manifest{ - {ID: "snap1", RootEntry: &kopia.DirEntry{ObjID: "obj1"}}, - {ID: "snap2", RootEntry: &kopia.DirEntry{ObjID: "obj2"}}, - }, - expected: []string{}, - }, - { - name: "Single pinned snapshot", - manifests: []kopia.Manifest{ - {ID: "snap1", RootEntry: &kopia.DirEntry{ObjID: "obj1"}, Pins: []string{"pin1"}}, - {ID: "snap2", RootEntry: &kopia.DirEntry{ObjID: "obj2"}}, - }, - expected: []string{"obj1"}, - }, - { - name: "Multiple pinned snapshots are sorted", - manifests: []kopia.Manifest{ - {ID: "snap-c", RootEntry: &kopia.DirEntry{ObjID: "obj-c"}, Pins: []string{"pin1"}}, - {ID: "snap-a", RootEntry: &kopia.DirEntry{ObjID: "obj-a"}, Pins: []string{"pin2"}}, - {ID: "snap-b", RootEntry: &kopia.DirEntry{ObjID: "obj-b"}}, - }, - expected: []string{"obj-a", "obj-c"}, - }, - { - name: "Duplicate ObjIDs are de-duplicated", - manifests: []kopia.Manifest{ - {ID: "snap1", RootEntry: &kopia.DirEntry{ObjID: "obj1"}, Pins: []string{"pin1"}}, - {ID: "snap2", RootEntry: &kopia.DirEntry{ObjID: "obj1"}, Pins: []string{"pin2"}}, - }, - expected: []string{"obj1"}, - }, - { - name: "Nil RootEntry with pins is skipped", - manifests: []kopia.Manifest{ - {ID: "snap1", RootEntry: nil, Pins: []string{"pin1"}}, - {ID: "snap2", RootEntry: &kopia.DirEntry{ObjID: "obj2"}, Pins: []string{"pin2"}}, - }, - expected: []string{"obj2"}, - }, - { - name: "Empty ObjID with pins is skipped", - manifests: []kopia.Manifest{ - {ID: "snap1", RootEntry: &kopia.DirEntry{ObjID: ""}, Pins: []string{"pin1"}}, - {ID: "snap2", RootEntry: &kopia.DirEntry{ObjID: "obj2"}, Pins: []string{"pin2"}}, - }, - expected: []string{"obj2"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := getPinnedSnapshots(tc.manifests) - - if !slices.Equal(got, tc.expected) { - t.Errorf("getPinnedSnapshots() = %v, want %v", got, tc.expected) - } - }) - } -} - func TestFindOldestWAL(t *testing.T) { tests := []struct { name string diff --git a/core/internal/consumer/maintenance.go b/core/internal/consumer/maintenance.go index c1a65278..36838bc4 100644 --- a/core/internal/consumer/maintenance.go +++ b/core/internal/consumer/maintenance.go @@ -25,9 +25,11 @@ import ( "strings" "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/cloudnative-pg/machinery/pkg/stringset" - "github.com/cloudnative-pg/klio/core/internal/kopia" + "github.com/cloudnative-pg/klio/core/internal/client/klioclient" "github.com/cloudnative-pg/klio/core/internal/opentelemetry" + "github.com/cloudnative-pg/klio/core/internal/queue" "github.com/cloudnative-pg/klio/core/internal/repository" ) @@ -35,23 +37,18 @@ import ( // that are no longer required by any remaining tier1 backup. It records the // tier1 maintenance metric; the error is returned for logging but is // best-effort (the caller does not fail the task on it). -// -// This is the server-side equivalent of the work that the `klio backup -// maintenance` command used to perform client-side after every backup. -func (d *Backup) maintainTier1(ctx context.Context, clusterName string, entries []kopia.Manifest) error { - if len(entries) == 0 { - return nil - } +func (d *Backup) maintainTier1(ctx context.Context, task *queue.BackupTask) error { + log.FromContext(ctx).Info("Applying tier1 maintenance", "cluster", task.ClusterName) - log.FromContext(ctx).Info("Applying tier1 maintenance", "cluster", clusterName) - - err := d.runTier1Retention(ctx, clusterName, entries) - recordMaintenance(ctx, clusterName, opentelemetry.Tier1, err) + err := d.runTier1Retention(ctx, task) + recordMaintenance(ctx, task.ClusterName, opentelemetry.Tier1, err) return err } -func (d *Backup) runTier1Retention(ctx context.Context, clusterName string, entries []kopia.Manifest) error { +func (d *Backup) runTier1Retention(ctx context.Context, task *queue.BackupTask) error { + clusterName := task.ClusterName + // The cluster name reaches us from the client's CloseBackup request via the // queue task and is used below as a WAL directory path, so validate it // before we touch the filesystem. This guard used to live in the gRPC @@ -60,20 +57,52 @@ func (d *Backup) runTier1Retention(ctx context.Context, clusterName string, entr return fmt.Errorf("invalid cluster name %q: %w", clusterName, err) } - userName := entries[0].Source.UserName + // Delete the tier1 base backups that fall outside the retention policy, + // while never deleting one that has not yet reached tier2, so no base backup + // is lost before it is durable on tier2. + keep, err := d.tier1RetentionGuard(ctx, clusterName) + if err != nil { + return err + } - // Apply the tier1 retention policy, deleting any base snapshots that are - // no longer needed. - if err := d.tier1Client.ApplyRetentionPolicy(ctx, kopia.Target{ - Username: userName, - Hostname: clusterName, - }); err != nil { + if err := d.applyRetention(ctx, d.tier1Client, clusterName, task.Tier1RetentionPolicy, keep); err != nil { return fmt.Errorf("while applying tier1 retention policy: %w", err) } return d.applyTier1WALRetention(ctx, clusterName) } +// tier1RetentionGuard returns a predicate that reports whether a tier1 backup +// must be kept because it has not yet been migrated to tier2. When tier2 is not +// configured there is nothing to protect and the predicate is nil. +func (d *Backup) tier1RetentionGuard(ctx context.Context, clusterName string) (func(name string) bool, error) { + if !d.tier2Enabled { + return nil, nil + } + + tier2Backups, err := d.tier2Client.ListBackups(ctx, clusterName) + if err != nil { + return nil, fmt.Errorf( + "while listing tier2 backups to guard tier1 retention for cluster %q: %w", clusterName, err) + } + + return keepUntilOnTier2(tier2Backups), nil +} + +// keepUntilOnTier2 returns a predicate that reports whether a tier1 backup must +// be kept because it is not yet present on tier2. A backup becomes deletable +// once it appears in the tier2 catalog. +func keepUntilOnTier2(tier2Backups klioclient.BackupList) func(name string) bool { + onTier2 := stringset.New() + for i := range tier2Backups { + onTier2.Put(tier2Backups[i].Name) + } + + return func(name string) bool { + return !onTier2.Has(name) + } +} + // applyTier1WALRetention drops the tier1 WAL files that are no longer required // by any remaining tier1 backup, clamped to the tier2 transfer frontier so // WALs still pending upload are never deleted. diff --git a/core/internal/consumer/maintenance_test.go b/core/internal/consumer/maintenance_test.go index 82dedad7..7af48367 100644 --- a/core/internal/consumer/maintenance_test.go +++ b/core/internal/consumer/maintenance_test.go @@ -19,7 +19,29 @@ SPDX-License-Identifier: Apache-2.0 package consumer -import "testing" +import ( + "testing" + + "github.com/cloudnative-pg/klio/core/internal/client/klioclient" +) + +func TestKeepUntilOnTier2(t *testing.T) { + tier2 := klioclient.BackupList{{Name: "backup-1"}, {Name: "backup-2"}} + keep := keepUntilOnTier2(tier2) + + // A backup already on tier2 is deletable (not kept). + if keep("backup-1") { + t.Error("keepUntilOnTier2() kept backup-1, which is already on tier2") + } + // A backup not yet on tier2 must be kept (protected from tier1 deletion). + if !keep("backup-3") { + t.Error("keepUntilOnTier2() did not keep backup-3, which is not on tier2") + } + // With no tier2 backups, everything is kept. + if keepAll := keepUntilOnTier2(nil); !keepAll("backup-1") { + t.Error("keepUntilOnTier2(nil) did not keep backup-1") + } +} func TestClampWAL(t *testing.T) { tests := []struct { diff --git a/core/internal/consumer/processbackup_test.go b/core/internal/consumer/processbackup_test.go index e4895f3d..4557324b 100644 --- a/core/internal/consumer/processbackup_test.go +++ b/core/internal/consumer/processbackup_test.go @@ -60,12 +60,12 @@ func (s *stubBackupSteps) relayTier2(_ context.Context, _ *queue.BackupTask, _ [ return s.relayErr } -func (s *stubBackupSteps) maintainTier2(_ context.Context, _ *queue.BackupTask, _ []kopia.Manifest) error { +func (s *stubBackupSteps) maintainTier2(_ context.Context, _ *queue.BackupTask) error { s.maintain2Called = true return s.maintain2Err } -func (s *stubBackupSteps) maintainTier1(_ context.Context, _ string, _ []kopia.Manifest) error { +func (s *stubBackupSteps) maintainTier1(_ context.Context, _ *queue.BackupTask) error { s.maintainCalled = true return s.maintainErr } diff --git a/core/internal/grpc/klio_wal.pb.go b/core/internal/grpc/klio_wal.pb.go index 9cc51d6d..7e53e5b8 100644 --- a/core/internal/grpc/klio_wal.pb.go +++ b/core/internal/grpc/klio_wal.pb.go @@ -730,8 +730,10 @@ type CloseBackupRequest struct { Tier2RetentionPolicy string `protobuf:"bytes,9,opt,name=tier2_retention_policy,json=tier2RetentionPolicy,proto3" json:"tier2_retention_policy,omitempty"` // When present, set the tier2 compression policy to the specified JSON-serialized policy. Tier2CompressionPolicy string `protobuf:"bytes,10,opt,name=tier2_compression_policy,json=tier2CompressionPolicy,proto3" json:"tier2_compression_policy,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // When present, set the tier1 retention policy to the specified JSON-serialized policy. + Tier1RetentionPolicy string `protobuf:"bytes,11,opt,name=tier1_retention_policy,json=tier1RetentionPolicy,proto3" json:"tier1_retention_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CloseBackupRequest) Reset() { @@ -827,6 +829,13 @@ func (x *CloseBackupRequest) GetTier2CompressionPolicy() string { return "" } +func (x *CloseBackupRequest) GetTier1RetentionPolicy() string { + if x != nil { + return x.Tier1RetentionPolicy + } + return "" +} + // This is sent by the WAL server in response to a CloseBackupRequest // message. type CloseBackupResult struct { @@ -930,7 +939,7 @@ const file_proto_klio_wal_proto_rawDesc = "" + "\fStartWALFile\x12!\n" + "\fklio_version\x18\x01 \x01(\x04R\vklioVersion\x12\x1f\n" + "\vfile_length\x18\x02 \x01(\x04R\n" + - "fileLength\"\xe1\x02\n" + + "fileLength\"\x97\x03\n" + "\x12CloseBackupRequest\x12!\n" + "\fcluster_name\x18\x01 \x01(\tR\vclusterName\x12\x1f\n" + "\vbackup_name\x18\x03 \x01(\tR\n" + @@ -942,7 +951,8 @@ const file_proto_klio_wal_proto_rawDesc = "" + "\rsend_to_tier2\x18\b \x01(\bR\vsendToTier2\x124\n" + "\x16tier2_retention_policy\x18\t \x01(\tR\x14tier2RetentionPolicy\x128\n" + "\x18tier2_compression_policy\x18\n" + - " \x01(\tR\x16tier2CompressionPolicy\"f\n" + + " \x01(\tR\x16tier2CompressionPolicy\x124\n" + + "\x16tier1_retention_policy\x18\v \x01(\tR\x14tier1RetentionPolicy\"f\n" + "\x11CloseBackupResult\x12%\n" + "\x0etier2_schedule\x18\x01 \x01(\bR\rtier2Schedule\x12*\n" + "\x11missing_wal_files\x18\x02 \x03(\tR\x0fmissingWalFiles2\xd8\x03\n" + diff --git a/core/internal/kopia/data.go b/core/internal/kopia/data.go index 0fafde97..a725e7dc 100644 --- a/core/internal/kopia/data.go +++ b/core/internal/kopia/data.go @@ -49,9 +49,6 @@ type Manifest struct { // Tags contains user-defined key-value pairs associated with the snapshot. Tags map[string]string `json:"tags,omitempty"` - - // Pins is a list of manually-defined pins which prevent the snapshot from being deleted. - Pins []string `json:"pins,omitempty"` } // DirEntry represents a directory entry as stored in JSON stream. @@ -129,39 +126,6 @@ func (ssi SourceInfo) String() string { return fmt.Sprintf("%v@%v:%v", ssi.UserName, ssi.Host, ssi.Path) } -// Policy describes snapshot policy for a single source. -type Policy struct { - // Labels contains key-value pairs associated with this policy. - Labels map[string]string `json:"-"` - - // RetentionPolicy defines how long snapshots should be retained. - RetentionPolicy RetentionPolicy `json:"retention"` - - // NoParent indicates whether this policy inherits from parent policies. - NoParent bool `json:"noParent,omitempty"` -} - -// RetentionPolicy describes snapshot retention policy. -type RetentionPolicy struct { - // KeepLatest is the number of most recent snapshots to keep. - KeepLatest *int `json:"keepLatest,omitempty"` - - // KeepHourly is the number of hourly snapshots to keep. - KeepHourly *int `json:"keepHourly,omitempty"` - - // KeepDaily is the number of daily snapshots to keep. - KeepDaily *int `json:"keepDaily,omitempty"` - - // KeepWeekly is the number of weekly snapshots to keep. - KeepWeekly *int `json:"keepWeekly,omitempty"` - - // KeepMonthly is the number of monthly snapshots to keep. - KeepMonthly *int `json:"keepMonthly,omitempty"` - - // KeepAnnual is the number of annual snapshots to keep. - KeepAnnual *int `json:"keepAnnual,omitempty"` -} - // CompressionPolicy describes the compression policy for a source. type CompressionPolicy struct { // Algorithm is the name of the Kopia compression algorithm to use. diff --git a/core/internal/kopia/data_test.go b/core/internal/kopia/data_test.go index c32879eb..ecce0d72 100644 --- a/core/internal/kopia/data_test.go +++ b/core/internal/kopia/data_test.go @@ -60,15 +60,3 @@ func TestSourceInfoString(t *testing.T) { }) } } - -func TestTargetString(t *testing.T) { - target := Target{ - Username: "user", - Hostname: "host", - } - - expected := "user@host" - if got := target.String(); got != expected { - t.Errorf("Target.Of() = %q, want %q", got, expected) - } -} diff --git a/core/internal/kopia/pins.go b/core/internal/kopia/pins.go deleted file mode 100644 index f0097757..00000000 --- a/core/internal/kopia/pins.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright © contributors to CloudNativePG, established as -CloudNativePG a Series of LF Projects, LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -*/ - -package kopia - -import ( - "context" - "fmt" - "os/exec" - - "github.com/cloudnative-pg/machinery/pkg/log" -) - -// PinSnapshotOpts contains options for pinning or unpinning snapshots. -type PinSnapshotOpts struct { - // The set of snapshots or root objects IDs to be pinned or unpinned. - IDs []string - - // AddPins is the list of pins to be added. - AddPins []string - - // RemovePins is the list of pins to be removed. - RemovePins []string -} - -// PinSnapshots pins and unpins a set of snapshots. -func (s *Client) PinSnapshots(ctx context.Context, opts PinSnapshotOpts) error { - if len(opts.IDs) == 0 { - return nil - } - - contextLogger := log.FromContext(ctx) - - args := make([]string, 0, 4+len(opts.AddPins)+len(opts.RemovePins)+len(opts.IDs)) - args = append(args, - "snapshot", - "pin", - "--config-file="+s.ConfigFile, - "--disable-file-logging", - ) - - for _, pin := range opts.AddPins { - args = append(args, "--add="+pin) - } - - for _, pin := range opts.RemovePins { - args = append(args, "--remove="+pin) - } - - args = append(args, opts.IDs...) - - contextLogger.Info("Pinning/unpinning Kopia snapshot", - "args", args, "addPins", opts.AddPins, "removePins", opts.RemovePins) - - pinSnapshotCmd := exec.CommandContext(ctx, s.KopiaBinary, args...) //nolint:gosec - pinSnapshotCmd.Env = s.kopiaEnvironmentVariables() - - if err := RunWithLogCapture(ctx, pinSnapshotCmd, nil); err != nil { - return fmt.Errorf("while pinning/unpinning Kopia snapshot: %w", err) - } - - return nil -} diff --git a/core/internal/kopia/policy.go b/core/internal/kopia/policy.go index 4fbcd17d..e66b6027 100644 --- a/core/internal/kopia/policy.go +++ b/core/internal/kopia/policy.go @@ -20,10 +20,8 @@ SPDX-License-Identifier: Apache-2.0 package kopia import ( - "bytes" "cmp" "context" - "encoding/json" "fmt" "os/exec" "strconv" @@ -31,83 +29,6 @@ import ( "github.com/cloudnative-pg/machinery/pkg/log" ) -// GetCurrentKopiaPolicy retrieves the current retention policy for a target. -func (s *Client) GetCurrentKopiaPolicy( - ctx context.Context, - t Target, -) (*Policy, error) { - contextLogger := log.FromContext(ctx) - - args := []string{ - "policy", - "show", - t.String(), - "--config-file=" + s.ConfigFile, - "--disable-file-logging", - "--json", - } - - contextLogger.Info("Getting Kopia policy", "args", args, "target", t) - - var buffer bytes.Buffer - - showPolicyCmd := exec.CommandContext(ctx, s.KopiaBinary, args...) //nolint:gosec - showPolicyCmd.Env = s.kopiaEnvironmentVariables() - - if err := RunWithLogCapture(ctx, showPolicyCmd, &buffer); err != nil { - return nil, fmt.Errorf("error while getting Kopia policy: %w", err) - } - - var result Policy - if err := json.NewDecoder(&buffer).Decode(&result); err != nil { - return nil, fmt.Errorf("cannot decode JSON backup metadata: %w", err) - } - - return &result, nil -} - -// SetKopiaPolicy sets the retention policy for a target. -func (s *Client) SetKopiaPolicy( - ctx context.Context, - t Target, - policy *RetentionPolicy, -) error { - policyToArgument := func(value *int) string { - if value == nil { - return "inherit" - } - - return strconv.Itoa(*value) - } - - contextLogger := log.FromContext(ctx) - - args := []string{ - "policy", - "set", - "--config-file=" + s.ConfigFile, - "--disable-file-logging", - "--keep-annual=" + policyToArgument(policy.KeepAnnual), - "--keep-daily=" + policyToArgument(policy.KeepDaily), - "--keep-hourly=" + policyToArgument(policy.KeepHourly), - "--keep-latest=" + policyToArgument(policy.KeepLatest), - "--keep-monthly=" + policyToArgument(policy.KeepMonthly), - "--keep-weekly=" + policyToArgument(policy.KeepWeekly), - t.String(), - } - - contextLogger.Info("Setting Kopia policy", "args", args, "target", t) - - setPolicyCmd := exec.CommandContext(ctx, s.KopiaBinary, args...) //nolint:gosec - setPolicyCmd.Env = s.kopiaEnvironmentVariables() - - if err := RunWithLogCapture(ctx, setPolicyCmd, nil); err != nil { - return fmt.Errorf("error while setting Kopia policy: %w", err) - } - - return nil -} - // SetKopiaCompressionPolicy sets the compression policy for a source. This // overrides the repository-wide global policy for that source. func (s *Client) SetKopiaCompressionPolicy( @@ -195,28 +116,3 @@ func compressionSizeArg(size int64) string { return strconv.FormatInt(size, 10) } - -// ApplyKopiaPolicy applies the retention policy by expiring old snapshots. -func (s *Client) ApplyKopiaPolicy(ctx context.Context, t Target) error { - contextLogger := log.FromContext(ctx) - - args := []string{ - "snapshot", - "expire", - "--config-file=" + s.ConfigFile, - "--disable-file-logging", - "--delete", - t.String(), - } - - contextLogger.Info("Applying Kopia policy", "args", args, "target", t) - - snapshotExpireCmd := exec.CommandContext(ctx, s.KopiaBinary, args...) //nolint:gosec - snapshotExpireCmd.Env = s.kopiaEnvironmentVariables() - - if err := RunWithLogCapture(ctx, snapshotExpireCmd, nil); err != nil { - return fmt.Errorf("error while applying Kopia policy: %w", err) - } - - return nil -} diff --git a/core/internal/kopia/write.go b/core/internal/kopia/write.go index 2e83fab8..1c2206f9 100644 --- a/core/internal/kopia/write.go +++ b/core/internal/kopia/write.go @@ -130,9 +130,6 @@ type SnapshotDirectoryOptions struct { // Description is a user-provided description of the snapshot. Description string - - // A pinned snapshot will not expire automatically. - Pins []string } // SnapshotDirectory creates a snapshot of a directory. @@ -155,10 +152,6 @@ func (s *Client) SnapshotDirectory( args = append(args, fmt.Sprintf("--tags=%s:%s", k, v)) } - for _, p := range opts.Pins { - args = append(args, "--pin="+p) - } - if opts.Description != "" { args = append(args, "--description="+opts.Description) } @@ -192,9 +185,6 @@ type SnapshotFileContentOptions struct { // Description is a user-provided description of the snapshot. Description string - - // A pinned snapshot will not expire automatically. - Pins []string } // SnapshotFileContent creates a snapshot from in-memory file content. @@ -217,10 +207,6 @@ func (s *Client) SnapshotFileContent( args = append(args, fmt.Sprintf("--tags=%s:%s", k, v)) } - for _, p := range opts.Pins { - args = append(args, "--pin="+p) - } - if opts.Description != "" { args = append(args, "--description="+opts.Description) } diff --git a/core/internal/queue/backup.go b/core/internal/queue/backup.go index 4f7b344d..fb08697b 100644 --- a/core/internal/queue/backup.go +++ b/core/internal/queue/backup.go @@ -26,6 +26,7 @@ import ( "github.com/nats-io/nats.go" "github.com/cloudnative-pg/klio/core/internal/kopia" + "github.com/cloudnative-pg/klio/core/pkg/retention" ) // BackupTask is the structure that is sent on NATS Stream when @@ -39,8 +40,13 @@ type BackupTask struct { // and maintenance) without touching tier2. SendToTier2 bool `json:"sendToTier2,omitempty"` - // The retention policy to apply to tier2. - Tier2RetentionPolicy *kopia.RetentionPolicy `json:"tier2RetentionPolicy,omitzero"` + // Tier1RetentionPolicy is the retention policy to apply to tier1. Its zero + // value keeps every backup. + Tier1RetentionPolicy retention.Policy `json:"tier1RetentionPolicy,omitzero"` + + // Tier2RetentionPolicy is the retention policy to apply to tier2. Its zero + // value keeps every backup. + Tier2RetentionPolicy retention.Policy `json:"tier2RetentionPolicy,omitzero"` // The compression policy to apply to tier2. Tier2CompressionPolicy *kopia.CompressionPolicy `json:"tier2CompressionPolicy,omitzero"` diff --git a/core/internal/server/admin/delete_backup_test.go b/core/internal/server/admin/delete_backup_test.go index c2519b9c..52111d2b 100644 --- a/core/internal/server/admin/delete_backup_test.go +++ b/core/internal/server/admin/delete_backup_test.go @@ -31,7 +31,6 @@ import ( "github.com/cloudnative-pg/klio/core/internal/client/klioclient" klioGRPC "github.com/cloudnative-pg/klio/core/internal/grpc" - "github.com/cloudnative-pg/klio/core/internal/kopia" ) // mockClient is a mock implementation of klioclient.Client for admin server tests. @@ -59,18 +58,6 @@ func (m *mockClient) GetMetadata(_ context.Context, _ string, _ string) (*kliocl return nil, nil } -func (m *mockClient) SetRetentionPolicy(_ context.Context, _ kopia.Target, _ kopia.RetentionPolicy) error { - return nil -} - -func (m *mockClient) GetRetentionPolicy(_ context.Context, _ kopia.Target) (*kopia.RetentionPolicy, error) { - return nil, nil -} - -func (m *mockClient) ApplyRetentionPolicy(_ context.Context, _ kopia.Target) error { - return nil -} - func (m *mockClient) GetUsername() string { return "" } func (m *mockClient) GetHostname() string { return "" } diff --git a/core/internal/server/walserver/backup.go b/core/internal/server/walserver/backup.go index fefe4d85..a47fbcca 100644 --- a/core/internal/server/walserver/backup.go +++ b/core/internal/server/walserver/backup.go @@ -32,6 +32,7 @@ import ( "github.com/cloudnative-pg/klio/core/internal/grpc" "github.com/cloudnative-pg/klio/core/internal/kopia" "github.com/cloudnative-pg/klio/core/internal/queue" + "github.com/cloudnative-pg/klio/core/pkg/retention" ) // CloseBackup implements the CloseBackup GRPC call. @@ -72,8 +73,6 @@ func (w *Implementation) CloseBackup( } func (w *Implementation) scheduleBackupRelay(ctx context.Context, request *grpc.CloseBackupRequest) error { - contextLogger := log.FromContext(ctx) - if w.queue == nil { return status.Errorf( codes.Internal, @@ -81,21 +80,14 @@ func (w *Implementation) scheduleBackupRelay(ctx context.Context, request *grpc. ) } - var tier2Policy *kopia.RetentionPolicy - if request.GetTier2RetentionPolicy() != "" { - var policy kopia.RetentionPolicy - if err := json.Unmarshal([]byte(request.GetTier2RetentionPolicy()), &policy); err != nil { - contextLogger.Error(err, "Unable to unmarshal tier2 retention policy, skipping") - } else { - tier2Policy = &policy - } - } + tier1Policy := parseRetentionPolicy(ctx, "tier1", request.GetTier1RetentionPolicy()) + tier2Policy := parseRetentionPolicy(ctx, "tier2", request.GetTier2RetentionPolicy()) var tier2Compression *kopia.CompressionPolicy if request.GetTier2CompressionPolicy() != "" { var policy kopia.CompressionPolicy if err := json.Unmarshal([]byte(request.GetTier2CompressionPolicy()), &policy); err != nil { - contextLogger.Error(err, "Unable to unmarshal tier2 compression policy, skipping") + log.FromContext(ctx).Error(err, "Unable to unmarshal tier2 compression policy, skipping") } else { tier2Compression = &policy } @@ -104,6 +96,7 @@ func (w *Implementation) scheduleBackupRelay(ctx context.Context, request *grpc. if err := w.queue.NotifyBackupReceived(ctx, &queue.BackupTask{ ClusterName: request.GetClusterName(), SendToTier2: request.GetSendToTier2(), + Tier1RetentionPolicy: tier1Policy, Tier2RetentionPolicy: tier2Policy, Tier2CompressionPolicy: tier2Compression, }); err != nil { @@ -113,6 +106,25 @@ func (w *Implementation) scheduleBackupRelay(ctx context.Context, request *grpc. return nil } +// parseRetentionPolicy decodes a JSON-serialized retention policy carried on a +// CloseBackup request. An empty payload, or one that fails to decode, yields the +// zero policy (keep everything); a decode error is logged and swallowed so a +// malformed policy never turns into an accidental mass deletion. +func parseRetentionPolicy(ctx context.Context, tier, payload string) retention.Policy { + if payload == "" { + return retention.Policy{} + } + + var policy retention.Policy + if err := json.Unmarshal([]byte(payload), &policy); err != nil { + log.FromContext(ctx).Error(err, "Unable to unmarshal retention policy, keeping everything", "tier", tier) + + return retention.Policy{} + } + + return policy +} + func (w *Implementation) checkWALFiles(request *grpc.CloseBackupRequest) ([]string, error) { startLSN, err := types.LSNStartFromWALName(request.GetStartWal(), request.GetSegmentSize()) if err != nil { diff --git a/core/internal/server/walserver/backup_test.go b/core/internal/server/walserver/backup_test.go new file mode 100644 index 00000000..94ce314d --- /dev/null +++ b/core/internal/server/walserver/backup_test.go @@ -0,0 +1,47 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package walserver + +import ( + "context" + "testing" + + "github.com/cloudnative-pg/klio/core/pkg/retention" +) + +func TestParseRetentionPolicy(t *testing.T) { + tests := []struct { + name string + payload string + want retention.Policy + }{ + {name: "empty payload keeps everything", payload: "", want: retention.Policy{}}, + {name: "valid policy is parsed", payload: `{"latest":5}`, want: retention.Policy{Latest: 5}}, + {name: "invalid JSON falls back to keep everything", payload: `{not json`, want: retention.Policy{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseRetentionPolicy(context.Background(), "tier1", tt.payload); got != tt.want { + t.Fatalf("parseRetentionPolicy() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/core/pkg/config/client_validate.go b/core/pkg/config/client_validate.go index 202c7b54..91da1f00 100644 --- a/core/pkg/config/client_validate.go +++ b/core/pkg/config/client_validate.go @@ -38,6 +38,18 @@ func (d *Data) Validate() error { errs = errors.Join(errs, err) } + if d.Tier1RetentionPolicy != nil { + if err := d.Tier1RetentionPolicy.Validate(); err != nil { + errs = errors.Join(errs, err) + } + } + + if d.Tier2RetentionPolicy != nil { + if err := d.Tier2RetentionPolicy.Validate(); err != nil { + errs = errors.Join(errs, err) + } + } + if err := d.Tier1CompressionPolicy.Validate(); err != nil { errs = errors.Join(errs, err) } diff --git a/core/pkg/config/retention.go b/core/pkg/config/retention.go index 6b21756f..244b63a8 100644 --- a/core/pkg/config/retention.go +++ b/core/pkg/config/retention.go @@ -19,23 +19,43 @@ SPDX-License-Identifier: Apache-2.0 package config -// RetentionPolicy defines how many backups we should keep. +import ( + "encoding/json" + "errors" + "fmt" +) + +// RetentionPolicy defines which backups Klio should keep. It is evaluated by +// the server against the backup catalog, replacing the retention that Kopia +// used to enforce on its own. type RetentionPolicy struct { - // KeepLatest is the number of latest backups to keep - KeepLatest *int `json:"keep_latest,omitempty" mapstructure:"keep_latest"` + // Latest keeps only the given number of most recent backups and deletes the + // rest. A configured policy always sets it to at least 1; keeping every + // backup is expressed by not configuring a retention policy at all. + Latest int `json:"latest,omitempty" mapstructure:"latest"` +} - // KeepAnnual is the number of annual backups to keep - KeepAnnual *int `json:"keep_annual,omitempty" mapstructure:"keep_annual"` +// Validate implements a custom validation function for RetentionPolicy. +func (r *RetentionPolicy) Validate() error { + if r.Latest < 1 { + return errors.New("invalid retention policy: latest must be greater than or equal to 1") + } - // KeepMonthly is the number of monthly backups to keep - KeepMonthly *int `json:"keep_monthly,omitempty" mapstructure:"keep_monthly"` + return nil +} - // KeepWeekly is the number of weekly backups to keep - KeepWeekly *int `json:"keep_weekly,omitempty" mapstructure:"keep_weekly"` +// MarshalWire serializes the policy to the JSON string used to carry it to the +// server. A nil policy yields an empty string, which the server reads as "keep +// everything". +func (r *RetentionPolicy) MarshalWire() (string, error) { + if r == nil { + return "", nil + } - // KeepDaily is the number of daily backups to keep - KeepDaily *int `json:"keep_daily,omitempty" mapstructure:"keep_daily"` + content, err := json.Marshal(r) + if err != nil { + return "", fmt.Errorf("while serializing retention policy: %w", err) + } - // KeepHourly is the number of hourly backups to keep - KeepHourly *int `json:"keep_hourly,omitempty" mapstructure:"keep_hourly"` + return string(content), nil } diff --git a/core/pkg/config/retention_test.go b/core/pkg/config/retention_test.go new file mode 100644 index 00000000..08a6dd3d --- /dev/null +++ b/core/pkg/config/retention_test.go @@ -0,0 +1,60 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package config + +import "testing" + +func TestRetentionPolicyValidate(t *testing.T) { + tests := []struct { + name string + policy RetentionPolicy + wantErr bool + }{ + {name: "latest 1 is valid", policy: RetentionPolicy{Latest: 1}}, + {name: "latest 10 is valid", policy: RetentionPolicy{Latest: 10}}, + {name: "latest 0 is invalid", policy: RetentionPolicy{Latest: 0}, wantErr: true}, + {name: "negative latest is invalid", policy: RetentionPolicy{Latest: -1}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.policy.Validate(); (err != nil) != tt.wantErr { + t.Fatalf("Validate() error = %v, wantErr = %v", err, tt.wantErr) + } + }) + } +} + +func TestRetentionPolicyMarshalWire(t *testing.T) { + t.Run("nil policy yields empty string", func(t *testing.T) { + var p *RetentionPolicy + got, err := p.MarshalWire() + if err != nil || got != "" { + t.Fatalf("MarshalWire() = %q, %v; want \"\", nil", got, err) + } + }) + + t.Run("configured policy serializes latest", func(t *testing.T) { + got, err := (&RetentionPolicy{Latest: 3}).MarshalWire() + if err != nil || got != `{"latest":3}` { + t.Fatalf("MarshalWire() = %q, %v; want %q, nil", got, err, `{"latest":3}`) + } + }) +} diff --git a/core/pkg/retention/retention.go b/core/pkg/retention/retention.go new file mode 100644 index 00000000..f553de97 --- /dev/null +++ b/core/pkg/retention/retention.go @@ -0,0 +1,88 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package retention evaluates Klio-managed retention policies against a backup +// catalog. It is intentionally free of any Kopia or I/O dependency so that the +// decision of which backups have expired can be tested in isolation and reused +// by other consumers of the same catalog format. +package retention + +import "sort" + +// Policy describes which backups Klio should keep. Its zero value means "no +// retention configured", which keeps every backup; a configured policy always +// carries a positive count, as values below 1 are rejected. +type Policy struct { + // Latest keeps only the N most recent backups and expires the rest. A value + // below 1 means no retention is configured, keeping everything. + Latest int `json:"latest,omitempty" mapstructure:"latest"` +} + +// Backup is the minimal view of a backup that retention evaluation needs. The +// catalog passed to Evaluate is a slice of these, decoupled from how the +// backups are actually stored. +type Backup struct { + // Name uniquely identifies the backup within a cluster. + Name string + + // StartedAt is the backup start time, in Unix seconds. It is the + // chronological key used to order the catalog. + StartedAt int64 + + // StoppedAt is the backup completion time, in Unix seconds. + StoppedAt int64 +} + +// Evaluate returns the backups in catalog that fall outside policy and should +// therefore be deleted. The input slice is not modified, and a backup is never +// returned more than once. +// +// For the "latest" criterion the N most recent backups (ordered by StartedAt, +// most recent first) are kept and every older backup is expired. A policy that +// keeps everything (see Policy.IsZero) yields no expired backups. +func Evaluate(catalog []Backup, policy Policy) []Backup { + // A count below 1 means no retention is configured (see Policy): keep + // everything. This also guards the slice bound below. + keep := policy.Latest + if keep < 1 { + return nil + } + + if len(catalog) <= keep { + return nil + } + + // Order a copy from most to least recent so the survivors are the newest + // backups. StartedAt is the primary key; the name breaks ties so the + // outcome is deterministic when two backups share a start time. + ordered := make([]Backup, len(catalog)) + copy(ordered, catalog) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].StartedAt != ordered[j].StartedAt { + return ordered[i].StartedAt > ordered[j].StartedAt + } + + return ordered[i].Name > ordered[j].Name + }) + + expired := make([]Backup, len(ordered)-keep) + copy(expired, ordered[keep:]) + + return expired +} diff --git a/core/pkg/retention/retention_test.go b/core/pkg/retention/retention_test.go new file mode 100644 index 00000000..e2ff2057 --- /dev/null +++ b/core/pkg/retention/retention_test.go @@ -0,0 +1,157 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package retention + +import ( + "reflect" + "sort" + "testing" +) + +// names extracts and sorts the backup names of a catalog so expectations can be +// compared regardless of the order Evaluate returns them in. +func names(backups []Backup) []string { + result := make([]string, 0, len(backups)) + for _, b := range backups { + result = append(result, b.Name) + } + sort.Strings(result) + + return result +} + +func TestEvaluateLatest(t *testing.T) { + catalog := []Backup{ + {Name: "b1", StartedAt: 100}, + {Name: "b2", StartedAt: 200}, + {Name: "b3", StartedAt: 300}, + {Name: "b4", StartedAt: 400}, + {Name: "b5", StartedAt: 500}, + } + + tests := []struct { + name string + catalog []Backup + policy Policy + wantExpired []string + }{ + { + name: "zero policy keeps everything", + catalog: catalog, + policy: Policy{}, + wantExpired: []string{}, + }, + { + name: "keep more than available expires nothing", + catalog: catalog, + policy: Policy{Latest: 10}, + wantExpired: []string{}, + }, + { + name: "keep exactly the catalog size expires nothing", + catalog: catalog, + policy: Policy{Latest: 5}, + wantExpired: []string{}, + }, + { + name: "keep the two most recent", + catalog: catalog, + policy: Policy{Latest: 2}, + wantExpired: []string{"b1", "b2", "b3"}, + }, + { + name: "keep only the most recent", + catalog: catalog, + policy: Policy{Latest: 1}, + wantExpired: []string{"b1", "b2", "b3", "b4"}, + }, + { + name: "empty catalog expires nothing", + catalog: nil, + policy: Policy{Latest: 2}, + wantExpired: []string{}, + }, + { + name: "negative count keeps everything defensively", + catalog: catalog, + policy: Policy{Latest: -1}, + wantExpired: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := names(Evaluate(tt.catalog, tt.policy)) + if !reflect.DeepEqual(got, tt.wantExpired) { + t.Fatalf("Evaluate() expired = %v, want %v", got, tt.wantExpired) + } + }) + } +} + +// TestEvaluateOrdersByStartTime makes sure the survivors are chosen by +// recency even when the catalog is supplied out of order. +func TestEvaluateOrdersByStartTime(t *testing.T) { + catalog := []Backup{ + {Name: "old", StartedAt: 100}, + {Name: "new", StartedAt: 300}, + {Name: "mid", StartedAt: 200}, + } + + got := names(Evaluate(catalog, Policy{Latest: 1})) + want := []string{"mid", "old"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Evaluate() expired = %v, want %v", got, want) + } +} + +// TestEvaluateDeterministicTie makes sure a shared start time yields a stable +// selection driven by the backup name. +func TestEvaluateDeterministicTie(t *testing.T) { + catalog := []Backup{ + {Name: "a", StartedAt: 100}, + {Name: "b", StartedAt: 100}, + {Name: "c", StartedAt: 100}, + } + + got := names(Evaluate(catalog, Policy{Latest: 1})) + // "c" sorts highest by name, so it survives; "a" and "b" expire. + want := []string{"a", "b"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Evaluate() expired = %v, want %v", got, want) + } +} + +// TestEvaluateDoesNotMutateInput guards against the copy-on-sort contract. +func TestEvaluateDoesNotMutateInput(t *testing.T) { + catalog := []Backup{ + {Name: "b1", StartedAt: 100}, + {Name: "b2", StartedAt: 200}, + {Name: "b3", StartedAt: 300}, + } + before := make([]Backup, len(catalog)) + copy(before, catalog) + + _ = Evaluate(catalog, Policy{Latest: 1}) + + if !reflect.DeepEqual(catalog, before) { + t.Fatalf("Evaluate() mutated its input: got %v, want %v", catalog, before) + } +} diff --git a/core/proto/klio_wal.proto b/core/proto/klio_wal.proto index 2d936e0c..02840971 100644 --- a/core/proto/klio_wal.proto +++ b/core/proto/klio_wal.proto @@ -164,6 +164,9 @@ message CloseBackupRequest { // When present, set the tier2 compression policy to the specified JSON-serialized policy. string tier2_compression_policy = 10; + + // When present, set the tier1 retention policy to the specified JSON-serialized policy. + string tier1_retention_policy = 11; } // This is sent by the WAL server in response to a CloseBackupRequest diff --git a/documentation/web/docs/developer/_protocol.md b/documentation/web/docs/developer/_protocol.md index 45aeaabe..e6638093 100644 --- a/documentation/web/docs/developer/_protocol.md +++ b/documentation/web/docs/developer/_protocol.md @@ -352,6 +352,7 @@ been completed. | send_to_tier2 | [bool](#bool) | | Require this backup to be sent to tier2. | | tier2_retention_policy | [string](#string) | | When present, set the tier2 retention policy to the specified JSON-serialized policy. | | tier2_compression_policy | [string](#string) | | When present, set the tier2 compression policy to the specified JSON-serialized policy. | +| tier1_retention_policy | [string](#string) | | When present, set the tier1 retention policy to the specified JSON-serialized policy. | diff --git a/documentation/web/docs/user/api/_klio_api.md b/documentation/web/docs/user/api/_klio_api.md index 2d32d506..c7d491c5 100644 --- a/documentation/web/docs/user/api/_klio_api.md +++ b/documentation/web/docs/user/api/_klio_api.md @@ -205,7 +205,8 @@ _Appears in:_ -RetentionPolicy defines how many backups we should keep. +RetentionPolicy defines which backups Klio should keep. Omitting the whole +policy keeps every backup. @@ -215,12 +216,7 @@ _Appears in:_ | Field | Description | Required | Default | Validation | | --- | --- | --- | --- | --- | -| `keepLatest` _integer_ | KeepLatest is the number of latest backups to keep
optional | True | | | -| `keepAnnual` _integer_ | KeepAnnual is the number of annual backups to keep
optional | True | | | -| `keepMonthly` _integer_ | KeepMonthly is the number of monthly backups to keep
optional | True | | | -| `keepWeekly` _integer_ | KeepWeekly is the number of weekly backups to keep
optional | True | | | -| `keepDaily` _integer_ | KeepDaily is the number of daily backups to keep
optional | True | | | -| `keepHourly` _integer_ | KeepHourly is the number of hourly backups to keep
optional | True | | | +| `latest` _integer_ | Latest keeps only the given number of most recent backups and deletes the
rest. | True | | | #### S3Configuration diff --git a/documentation/web/docs/user/cli/klio.md b/documentation/web/docs/user/cli/klio.md index a8372388..6febaf61 100644 --- a/documentation/web/docs/user/cli/klio.md +++ b/documentation/web/docs/user/cli/klio.md @@ -43,7 +43,6 @@ run directly by end users. * [klio get-wal](klio_get-wal.md) - Get a WAL from the target Klio server * [klio reset-lsn](klio_reset-lsn.md) - Reset the replication status to the latest flush LSN * [klio restore](klio_restore.md) - Restore a PostgreSQL cluster from a Klio server -* [klio retention](klio_retention.md) - Manage the retention policy * [klio send-wal](klio_send-wal.md) - Upload the cluster's WALs to the target Klio server * [klio server](klio_server.md) - Starts and manage a Klio server * [klio wal-player](klio_wal-player.md) - WAL Player Commands diff --git a/documentation/web/docs/user/cli/klio_retention.md b/documentation/web/docs/user/cli/klio_retention.md deleted file mode 100644 index 7a0776cb..00000000 --- a/documentation/web/docs/user/cli/klio_retention.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: klio retention ---- - -## klio retention - -Manage the retention policy - -### Options - -``` - -h, --help help for retention -``` - -### Options inherited from parent commands - -``` - --config string config file (default is $HOME/.klio.yaml) - --debug enable debug logging - --log-destination string where the log stream will be written - --log-field-level string JSON log field to report severity in (default: level) - --log-field-timestamp string JSON log field to report timestamp in (default: ts) - --log-level string the desired log level, one of error, info, debug and trace (default "info") - --log-truncate-destination truncate the log destination on open instead of appending to it (ignored for FIFOs) - --pprof-server string enable the PPROF server using the specified address - --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) - --zap-encoder encoder Zap log encoding (one of 'json' or 'console') - --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' or any integer value > 0 which corresponds to custom debug levels of increasing verbosity - --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). - --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. -``` - -### SEE ALSO - -* [klio](klio.md) - PostgreSQL Backup & Recovery for CloudNativePG -* [klio retention get](klio_retention_get.md) - Gets the currently applied retention policy -* [klio retention set](klio_retention_set.md) - Sets the currently applied retention policy - diff --git a/documentation/web/docs/user/cli/klio_retention_get.md b/documentation/web/docs/user/cli/klio_retention_get.md deleted file mode 100644 index 2b08af3b..00000000 --- a/documentation/web/docs/user/cli/klio_retention_get.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: klio retention get ---- - -## klio retention get - -Gets the currently applied retention policy - -``` -klio retention get [flags] -``` - -### Options - -``` - -h, --help help for get -``` - -### Options inherited from parent commands - -``` - --config string config file (default is $HOME/.klio.yaml) - --debug enable debug logging - --log-destination string where the log stream will be written - --log-field-level string JSON log field to report severity in (default: level) - --log-field-timestamp string JSON log field to report timestamp in (default: ts) - --log-level string the desired log level, one of error, info, debug and trace (default "info") - --log-truncate-destination truncate the log destination on open instead of appending to it (ignored for FIFOs) - --pprof-server string enable the PPROF server using the specified address - --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) - --zap-encoder encoder Zap log encoding (one of 'json' or 'console') - --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' or any integer value > 0 which corresponds to custom debug levels of increasing verbosity - --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). - --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. -``` - -### SEE ALSO - -* [klio retention](klio_retention.md) - Manage the retention policy - diff --git a/documentation/web/docs/user/cli/klio_retention_set.md b/documentation/web/docs/user/cli/klio_retention_set.md deleted file mode 100644 index 0f0f5cad..00000000 --- a/documentation/web/docs/user/cli/klio_retention_set.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: klio retention set ---- - -## klio retention set - -Sets the currently applied retention policy - -``` -klio retention set [flags] -``` - -### Options - -``` - -h, --help help for set - --keep-annual int Number of most recent annual backup kept - --keep-daily int Number of most recent daily backup kept - --keep-hourly int Number of most recent hourly backup kept - --keep-latest int Number of most recent latest backup kept - --keep-monthly int Number of most recent monthly backup kept - --keep-weekly int Number of most recent weekly backup kept -``` - -### Options inherited from parent commands - -``` - --config string config file (default is $HOME/.klio.yaml) - --debug enable debug logging - --log-destination string where the log stream will be written - --log-field-level string JSON log field to report severity in (default: level) - --log-field-timestamp string JSON log field to report timestamp in (default: ts) - --log-level string the desired log level, one of error, info, debug and trace (default "info") - --log-truncate-destination truncate the log destination on open instead of appending to it (ignored for FIFOs) - --pprof-server string enable the PPROF server using the specified address - --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) - --zap-encoder encoder Zap log encoding (one of 'json' or 'console') - --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' or any integer value > 0 which corresponds to custom debug levels of increasing verbosity - --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). - --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. -``` - -### SEE ALSO - -* [klio retention](klio_retention.md) - Manage the retention policy - diff --git a/documentation/web/docs/user/concepts/architectures.md b/documentation/web/docs/user/concepts/architectures.md index 74187f6d..82a7d7e9 100644 --- a/documentation/web/docs/user/concepts/architectures.md +++ b/documentation/web/docs/user/concepts/architectures.md @@ -139,26 +139,17 @@ CloudNativePG topology, this allows a Klio server at a secondary site to use the shared Tier 2 storage to bootstrap a new cluster, enhancing DR capabilities. -### Snapshot Pinning - -When Tier 2 is enabled, Klio automatically pins snapshots in Tier 1 with a -`klio.io/tier2` pin. This mechanism prevents retention policies from -automatically deleting snapshots before they have been successfully migrated -to Tier 2. - -The pinning workflow operates as follows: - -1. When a backup is created with Tier 2 enabled, all snapshot components - (tablespaces, PGDATA, control file, and metadata) are tagged with the - `klio.io/tier2` pin. -1. The pin protects the snapshot from being removed by retention policy - enforcement, even if it would otherwise be eligible for deletion. -1. After the snapshot is successfully migrated to Tier 2, Klio removes the - pin, allowing normal retention policy management to resume. - -This ensures data integrity during the asynchronous migration process and -guarantees that no backup is lost due to retention policies running before -migration completes. +### Protecting backups pending Tier 2 migration + +When Tier 2 is enabled, a backup is written to Tier 1 first and then migrated +to Tier 2 asynchronously. During that window the Klio server never deletes a +Tier 1 backup that has not yet reached Tier 2: when it applies the Tier 1 +retention policy, it skips any backup that is not present in the Tier 2 +catalog. + +This guarantees that no backup is lost due to retention running before the +migration completes, and it is enforced entirely from Klio's own backup +catalog. ### Restoring from Tier 2 diff --git a/documentation/web/docs/user/plugin_configuration.md b/documentation/web/docs/user/plugin_configuration.md index a0243f42..3f4493cd 100644 --- a/documentation/web/docs/user/plugin_configuration.md +++ b/documentation/web/docs/user/plugin_configuration.md @@ -209,7 +209,7 @@ customize the plugin's behavior. ### Retention policies -Define how long backups should be retained by configuring retention policies +Define which backups should be retained by configuring retention policies for Tier 1 and Tier 2 storage. Retention policies can be configured independently for each tier: @@ -225,47 +225,23 @@ spec: clusterName: cluster-example tier1: retention: - keepLatest: 5 - keepHourly: 12 - keepDaily: 7 - keepWeekly: 4 - keepMonthly: 6 - keepAnnual: 2 + latest: 5 tier2: enableBackup: true enableRecovery: true retention: - keepLatest: 10 - keepDaily: 30 - keepMonthly: 12 - keepAnnual: 5 + latest: 10 ``` -Except for `keepLatest`, each option defines how many backups to retain -for the specified time period. For example, `keepDaily: 7` means that we should -retain at most one backup for each of the past 7 days. - -If multiple backups exist within the same time bucket, the most recent one is -kept, unless preserved by a different *keep* rule. Backups that are not -retained by any rule are deleted. Rule evaluation is done when a new backup is -taken. +The `latest` option keeps only the given number of most recent backups and +deletes the rest. Retention is evaluated by the Klio server against its own +backup catalog every time a new backup is taken. The Klio server will automatically delete WAL files that are no longer needed for recovery by any retained backup. -All retention settings are optional. For each unspecified retention level, -the default Kopia value is applied: - -```yaml -keepLatest: 10 -keepHourly: 48 -keepDaily: 7 -keepWeekly: 4 -keepMonthly: 24 -keepAnnual: 1 -``` - -Set a rule to `0` to disable that retention level. +The retention policy is optional and must be set to at least `1` when present. +Omit it entirely to keep every backup. ### Compression policies @@ -457,8 +433,7 @@ spec: enableBackup: true enableRecovery: true retention: - keepDaily: 30 - keepMonthly: 12 + latest: 10 ``` #### Options diff --git a/documentation/web/docs/user/upgrade_notes.md b/documentation/web/docs/user/upgrade_notes.md index 90f3f6e1..41b0b78a 100644 --- a/documentation/web/docs/user/upgrade_notes.md +++ b/documentation/web/docs/user/upgrade_notes.md @@ -8,6 +8,22 @@ This page lists version-specific changes that may require manual action when upgrading Klio. For the upgrade procedure, see the [Helm chart page](helm_chart.mdx#upgrades). +## Unreleased + +### Klio-managed retention policies + +Retention is now evaluated by the Klio server against its own backup catalog +instead of being delegated to Kopia. + +- The `retention` block of the `PluginConfiguration` changed shape. The + Kopia-style `keepLatest`, `keepHourly`, `keepDaily`, `keepWeekly`, + `keepMonthly` and `keepAnnual` fields are replaced by a single `latest` + field, which keeps the given number of most recent backups. Update any + `tier1.retention` and `tier2.retention` blocks accordingly. Omit the block to + keep every backup; when present, `latest` must be at least `1`. +- The `klio retention` command has been removed. Retention is configured only + through the `PluginConfiguration`. + ## 0.0.20 to 0.0.21 ### Migrating from the Multi-PVC Model diff --git a/operator/api/v1alpha1/plugin_configuration_types.go b/operator/api/v1alpha1/plugin_configuration_types.go index 6c3c8f61..92af09d1 100644 --- a/operator/api/v1alpha1/plugin_configuration_types.go +++ b/operator/api/v1alpha1/plugin_configuration_types.go @@ -205,31 +205,13 @@ func (s *PluginConfigurationSpec) GetWALPrefetch() WALPrefetchConfiguration { return result } -// RetentionPolicy defines how many backups we should keep. +// RetentionPolicy defines which backups Klio should keep. Omitting the whole +// policy keeps every backup. +// +kubebuilder:validation:XValidation:rule="self.latest >= 1",message="latest must be greater than or equal to 1" type RetentionPolicy struct { - // KeepLatest is the number of latest backups to keep - // optional - KeepLatest *int `json:"keepLatest,omitempty" mapstructure:"keepLatest"` - - // KeepAnnual is the number of annual backups to keep - // optional - KeepAnnual *int `json:"keepAnnual,omitempty" mapstructure:"keepAnnual"` - - // KeepMonthly is the number of monthly backups to keep - // optional - KeepMonthly *int `json:"keepMonthly,omitempty" mapstructure:"keepMonthly"` - - // KeepWeekly is the number of weekly backups to keep - // optional - KeepWeekly *int `json:"keepWeekly,omitempty" mapstructure:"keepWeekly"` - - // KeepDaily is the number of daily backups to keep - // optional - KeepDaily *int `json:"keepDaily,omitempty" mapstructure:"keepDaily"` - - // KeepHourly is the number of hourly backups to keep - // optional - KeepHourly *int `json:"keepHourly,omitempty" mapstructure:"keepHourly"` + // Latest keeps only the given number of most recent backups and deletes the + // rest. + Latest int `json:"latest" mapstructure:"latest"` } // PluginConfigurationStatus defines the observed state of ClientConfig. diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 9ece6c0d..7e660fd6 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -268,36 +268,6 @@ func (in *PodTemplateSpec) DeepCopy() *PodTemplateSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RetentionPolicy) DeepCopyInto(out *RetentionPolicy) { *out = *in - if in.KeepLatest != nil { - in, out := &in.KeepLatest, &out.KeepLatest - *out = new(int) - **out = **in - } - if in.KeepAnnual != nil { - in, out := &in.KeepAnnual, &out.KeepAnnual - *out = new(int) - **out = **in - } - if in.KeepMonthly != nil { - in, out := &in.KeepMonthly, &out.KeepMonthly - *out = new(int) - **out = **in - } - if in.KeepWeekly != nil { - in, out := &in.KeepWeekly, &out.KeepWeekly - *out = new(int) - **out = **in - } - if in.KeepDaily != nil { - in, out := &in.KeepDaily, &out.KeepDaily - *out = new(int) - **out = **in - } - if in.KeepHourly != nil { - in, out := &in.KeepHourly, &out.KeepHourly - *out = new(int) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RetentionPolicy. @@ -511,7 +481,7 @@ func (in *Tier1PluginConfiguration) DeepCopyInto(out *Tier1PluginConfiguration) if in.RetentionPolicy != nil { in, out := &in.RetentionPolicy, &out.RetentionPolicy *out = new(RetentionPolicy) - (*in).DeepCopyInto(*out) + **out = **in } if in.Compression != nil { in, out := &in.Compression, &out.Compression @@ -563,7 +533,7 @@ func (in *Tier2PluginConfiguration) DeepCopyInto(out *Tier2PluginConfiguration) if in.RetentionPolicy != nil { in, out := &in.RetentionPolicy, &out.RetentionPolicy *out = new(RetentionPolicy) - (*in).DeepCopyInto(*out) + **out = **in } if in.Compression != nil { in, out := &in.Compression, &out.Compression diff --git a/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml b/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml index 8f5664f6..db11bc80 100644 --- a/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml +++ b/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml @@ -1727,37 +1727,17 @@ spec: description: RetentionPolicy defines how many backups we should keep properties: - keepAnnual: + latest: description: |- - KeepAnnual is the number of annual backups to keep - optional - type: integer - keepDaily: - description: |- - KeepDaily is the number of daily backups to keep - optional - type: integer - keepHourly: - description: |- - KeepHourly is the number of hourly backups to keep - optional - type: integer - keepLatest: - description: |- - KeepLatest is the number of latest backups to keep - optional - type: integer - keepMonthly: - description: |- - KeepMonthly is the number of monthly backups to keep - optional - type: integer - keepWeekly: - description: |- - KeepWeekly is the number of weekly backups to keep - optional + Latest keeps only the given number of most recent backups and deletes the + rest. type: integer + required: + - latest type: object + x-kubernetes-validations: + - message: latest must be greater than or equal to 1 + rule: self.latest >= 1 type: object tier2: description: Tier2 is the Tier 2 configuration @@ -1823,37 +1803,17 @@ spec: description: RetentionPolicy defines how many backups we should keep properties: - keepAnnual: + latest: description: |- - KeepAnnual is the number of annual backups to keep - optional - type: integer - keepDaily: - description: |- - KeepDaily is the number of daily backups to keep - optional - type: integer - keepHourly: - description: |- - KeepHourly is the number of hourly backups to keep - optional - type: integer - keepLatest: - description: |- - KeepLatest is the number of latest backups to keep - optional - type: integer - keepMonthly: - description: |- - KeepMonthly is the number of monthly backups to keep - optional - type: integer - keepWeekly: - description: |- - KeepWeekly is the number of weekly backups to keep - optional + Latest keeps only the given number of most recent backups and deletes the + rest. type: integer + required: + - latest type: object + x-kubernetes-validations: + - message: latest must be greater than or equal to 1 + rule: self.latest >= 1 type: object x-kubernetes-validations: - message: at least one of enableBackup or enableRecovery must be diff --git a/operator/dist/chart/crds/pluginconfiguration-crd.yaml b/operator/dist/chart/crds/pluginconfiguration-crd.yaml index 505840c3..c07f6dcf 100644 --- a/operator/dist/chart/crds/pluginconfiguration-crd.yaml +++ b/operator/dist/chart/crds/pluginconfiguration-crd.yaml @@ -1726,37 +1726,17 @@ spec: description: RetentionPolicy defines how many backups we should keep properties: - keepAnnual: + latest: description: |- - KeepAnnual is the number of annual backups to keep - optional - type: integer - keepDaily: - description: |- - KeepDaily is the number of daily backups to keep - optional - type: integer - keepHourly: - description: |- - KeepHourly is the number of hourly backups to keep - optional - type: integer - keepLatest: - description: |- - KeepLatest is the number of latest backups to keep - optional - type: integer - keepMonthly: - description: |- - KeepMonthly is the number of monthly backups to keep - optional - type: integer - keepWeekly: - description: |- - KeepWeekly is the number of weekly backups to keep - optional + Latest keeps only the given number of most recent backups and deletes the + rest. type: integer + required: + - latest type: object + x-kubernetes-validations: + - message: latest must be greater than or equal to 1 + rule: self.latest >= 1 type: object tier2: description: Tier2 is the Tier 2 configuration @@ -1822,37 +1802,17 @@ spec: description: RetentionPolicy defines how many backups we should keep properties: - keepAnnual: + latest: description: |- - KeepAnnual is the number of annual backups to keep - optional - type: integer - keepDaily: - description: |- - KeepDaily is the number of daily backups to keep - optional - type: integer - keepHourly: - description: |- - KeepHourly is the number of hourly backups to keep - optional - type: integer - keepLatest: - description: |- - KeepLatest is the number of latest backups to keep - optional - type: integer - keepMonthly: - description: |- - KeepMonthly is the number of monthly backups to keep - optional - type: integer - keepWeekly: - description: |- - KeepWeekly is the number of weekly backups to keep - optional + Latest keeps only the given number of most recent backups and deletes the + rest. type: integer + required: + - latest type: object + x-kubernetes-validations: + - message: latest must be greater than or equal to 1 + rule: self.latest >= 1 type: object x-kubernetes-validations: - message: at least one of enableBackup or enableRecovery must be diff --git a/operator/internal/klioconfig/config.go b/operator/internal/klioconfig/config.go index 0c4143e4..79dc36d8 100644 --- a/operator/internal/klioconfig/config.go +++ b/operator/internal/klioconfig/config.go @@ -162,12 +162,7 @@ func convertRetentionPolicy(p *kliov1alpha1.RetentionPolicy) *config.RetentionPo } return &config.RetentionPolicy{ - KeepLatest: p.KeepLatest, - KeepAnnual: p.KeepAnnual, - KeepMonthly: p.KeepMonthly, - KeepWeekly: p.KeepWeekly, - KeepDaily: p.KeepDaily, - KeepHourly: p.KeepHourly, + Latest: p.Latest, } } diff --git a/operator/internal/klioconfig/config_test.go b/operator/internal/klioconfig/config_test.go index 0a4d70ae..286c9558 100644 --- a/operator/internal/klioconfig/config_test.go +++ b/operator/internal/klioconfig/config_test.go @@ -227,8 +227,7 @@ func TestGenerateConfig(t *testing.T) { ClusterName: testClusterName, Tier1: &kliov1alpha1.Tier1PluginConfiguration{ RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(5), - KeepDaily: new(7), + Latest: 5, }, }, }, @@ -236,8 +235,7 @@ func TestGenerateConfig(t *testing.T) { assertions: func(t *testing.T, cfg *config.Data) { t.Helper() assert.NotNil(t, cfg.Tier1RetentionPolicy) - assert.Equal(t, new(5), cfg.Tier1RetentionPolicy.KeepLatest) - assert.Equal(t, new(7), cfg.Tier1RetentionPolicy.KeepDaily) + assert.Equal(t, 5, cfg.Tier1RetentionPolicy.Latest) }, }, { @@ -249,8 +247,7 @@ func TestGenerateConfig(t *testing.T) { Tier2: &kliov1alpha1.Tier2PluginConfiguration{ EnableBackup: true, RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepWeekly: new(4), - KeepMonthly: new(12), + Latest: 12, }, }, }, @@ -258,8 +255,7 @@ func TestGenerateConfig(t *testing.T) { assertions: func(t *testing.T, cfg *config.Data) { t.Helper() assert.NotNil(t, cfg.Tier2RetentionPolicy) - assert.Equal(t, new(4), cfg.Tier2RetentionPolicy.KeepWeekly) - assert.Equal(t, new(12), cfg.Tier2RetentionPolicy.KeepMonthly) + assert.Equal(t, 12, cfg.Tier2RetentionPolicy.Latest) }, }, { @@ -270,13 +266,13 @@ func TestGenerateConfig(t *testing.T) { ClusterName: testClusterName, Tier1: &kliov1alpha1.Tier1PluginConfiguration{ RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(3), + Latest: 3, }, }, Tier2: &kliov1alpha1.Tier2PluginConfiguration{ EnableBackup: true, RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(10), + Latest: 10, }, }, }, @@ -285,8 +281,8 @@ func TestGenerateConfig(t *testing.T) { t.Helper() assert.NotNil(t, cfg.Tier1RetentionPolicy) assert.NotNil(t, cfg.Tier2RetentionPolicy) - assert.Equal(t, new(3), cfg.Tier1RetentionPolicy.KeepLatest) - assert.Equal(t, new(10), cfg.Tier2RetentionPolicy.KeepLatest) + assert.Equal(t, 3, cfg.Tier1RetentionPolicy.Latest) + assert.Equal(t, 10, cfg.Tier2RetentionPolicy.Latest) }, }, { @@ -345,45 +341,18 @@ func TestConvertRetentionPolicy(t *testing.T) { assert.Nil(t, result) }) - t.Run("all fields set", func(t *testing.T) { + t.Run("latest is set", func(t *testing.T) { input := &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(5), - KeepAnnual: new(2), - KeepMonthly: new(6), - KeepWeekly: new(4), - KeepDaily: new(7), - KeepHourly: new(24), + Latest: 5, } result := convertRetentionPolicy(input) assert.NotNil(t, result) assert.Equal(t, &config.RetentionPolicy{ - KeepLatest: new(5), - KeepAnnual: new(2), - KeepMonthly: new(6), - KeepWeekly: new(4), - KeepDaily: new(7), - KeepHourly: new(24), + Latest: 5, }, result) }) - - t.Run("partial fields set", func(t *testing.T) { - input := &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(3), - KeepDaily: new(7), - } - - result := convertRetentionPolicy(input) - - assert.NotNil(t, result) - assert.Equal(t, new(3), result.KeepLatest) - assert.Equal(t, new(7), result.KeepDaily) - assert.Nil(t, result.KeepAnnual) - assert.Nil(t, result.KeepMonthly) - assert.Nil(t, result.KeepWeekly) - assert.Nil(t, result.KeepHourly) - }) } func TestConvertTier1RetentionPolicy(t *testing.T) { @@ -401,14 +370,14 @@ func TestConvertTier1RetentionPolicy(t *testing.T) { t.Run("tier1 with retention policy", func(t *testing.T) { tier1 := &kliov1alpha1.Tier1PluginConfiguration{ RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(10), + Latest: 10, }, } result := convertTier1RetentionPolicy(tier1) assert.NotNil(t, result) - assert.Equal(t, new(10), result.KeepLatest) + assert.Equal(t, 10, result.Latest) }) } @@ -488,16 +457,14 @@ func TestConvertTier2RetentionPolicy(t *testing.T) { tier2 := &kliov1alpha1.Tier2PluginConfiguration{ EnableBackup: true, RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepDaily: new(7), - KeepWeekly: new(4), + Latest: 7, }, } result := convertTier2RetentionPolicy(tier2) assert.NotNil(t, result) - assert.Equal(t, new(7), result.KeepDaily) - assert.Equal(t, new(4), result.KeepWeekly) + assert.Equal(t, 7, result.Latest) }) } diff --git a/operator/pkg/config/retention.go b/operator/pkg/config/retention.go index 6b21756f..244b63a8 100644 --- a/operator/pkg/config/retention.go +++ b/operator/pkg/config/retention.go @@ -19,23 +19,43 @@ SPDX-License-Identifier: Apache-2.0 package config -// RetentionPolicy defines how many backups we should keep. +import ( + "encoding/json" + "errors" + "fmt" +) + +// RetentionPolicy defines which backups Klio should keep. It is evaluated by +// the server against the backup catalog, replacing the retention that Kopia +// used to enforce on its own. type RetentionPolicy struct { - // KeepLatest is the number of latest backups to keep - KeepLatest *int `json:"keep_latest,omitempty" mapstructure:"keep_latest"` + // Latest keeps only the given number of most recent backups and deletes the + // rest. A configured policy always sets it to at least 1; keeping every + // backup is expressed by not configuring a retention policy at all. + Latest int `json:"latest,omitempty" mapstructure:"latest"` +} - // KeepAnnual is the number of annual backups to keep - KeepAnnual *int `json:"keep_annual,omitempty" mapstructure:"keep_annual"` +// Validate implements a custom validation function for RetentionPolicy. +func (r *RetentionPolicy) Validate() error { + if r.Latest < 1 { + return errors.New("invalid retention policy: latest must be greater than or equal to 1") + } - // KeepMonthly is the number of monthly backups to keep - KeepMonthly *int `json:"keep_monthly,omitempty" mapstructure:"keep_monthly"` + return nil +} - // KeepWeekly is the number of weekly backups to keep - KeepWeekly *int `json:"keep_weekly,omitempty" mapstructure:"keep_weekly"` +// MarshalWire serializes the policy to the JSON string used to carry it to the +// server. A nil policy yields an empty string, which the server reads as "keep +// everything". +func (r *RetentionPolicy) MarshalWire() (string, error) { + if r == nil { + return "", nil + } - // KeepDaily is the number of daily backups to keep - KeepDaily *int `json:"keep_daily,omitempty" mapstructure:"keep_daily"` + content, err := json.Marshal(r) + if err != nil { + return "", fmt.Errorf("while serializing retention policy: %w", err) + } - // KeepHourly is the number of hourly backups to keep - KeepHourly *int `json:"keep_hourly,omitempty" mapstructure:"keep_hourly"` + return string(content), nil } diff --git a/operator/test/e2e/pluginconfiguration_update_test.go b/operator/test/e2e/pluginconfiguration_update_test.go index 635e2311..917dfefc 100644 --- a/operator/test/e2e/pluginconfiguration_update_test.go +++ b/operator/test/e2e/pluginconfiguration_update_test.go @@ -145,7 +145,7 @@ func (c *pluginConfigurationUpdateScenario) Run( if currentPC.Spec.Tier1.RetentionPolicy == nil { currentPC.Spec.Tier1.RetentionPolicy = &kliov1alpha1.RetentionPolicy{} } - currentPC.Spec.Tier1.RetentionPolicy.KeepLatest = new(5) + currentPC.Spec.Tier1.RetentionPolicy.Latest = 5 require.NoError(t, r.Update(ctx, ¤tPC), "failed to update PluginConfiguration") diff --git a/operator/test/e2e/tier2_retention_test.go b/operator/test/e2e/tier2_retention_test.go index 2558a33a..f93d6f15 100644 --- a/operator/test/e2e/tier2_retention_test.go +++ b/operator/test/e2e/tier2_retention_test.go @@ -271,12 +271,7 @@ func NewTier2RetentionFeatureConfig( EnableTier2Recovery: false, Mode: kliov1alpha1.ModeStandard, Tier2RetentionPolicy: &kliov1alpha1.RetentionPolicy{ - KeepLatest: new(tier2RetentionKeepNum), - KeepHourly: new(0), - KeepDaily: new(0), - KeepWeekly: new(0), - KeepMonthly: new(0), - KeepAnnual: new(0), + Latest: tier2RetentionKeepNum, }, }, ) @@ -325,9 +320,10 @@ func NewTier2RetentionFeatureConfig( } // Tier2Retention returns a Tier2RetentionFeature for testing tier2 retention. -// This test validates both backup retention (Kopia snapshots kept to keepLatest=1) -// and WAL retention (cleanup of WALs older than the oldest remaining backup). +// It keeps the two most recent backups, takes three, and verifies the oldest is +// deleted; it then tightens the policy to one and applies it on demand with +// `klio retention apply`. It also validates WAL retention. func Tier2Retention(namespace string) *klioFeatures.Tier2RetentionFeature { return klioFeatures.NewTier2RetentionFeature( - NewTier2RetentionFeatureConfig("Tier2Retention", namespace, 1)) + NewTier2RetentionFeatureConfig("Tier2Retention", namespace, 2)) } diff --git a/operator/test/klio/features/tier2_retention.go b/operator/test/klio/features/tier2_retention.go index ee774eb9..f33a3145 100644 --- a/operator/test/klio/features/tier2_retention.go +++ b/operator/test/klio/features/tier2_retention.go @@ -21,10 +21,11 @@ package features import ( "bytes" + "cmp" "context" "encoding/json" - "errors" "fmt" + "slices" "strings" "testing" "time" @@ -51,10 +52,6 @@ const ( tier2AnnotationName = "klio.io/tier2" // presentAnnotationValue is the value set when a backup is present in a tier. presentAnnotationValue = "present" - // tier2KopiaConfigPattern is the glob pattern for finding the tier2 Kopia config file. - // Used by verifyTier2RetentionPolicySet to run kopia policy commands. - // The file ending in .kopia-password contains the path to the actual config. - tier2KopiaConfigPattern = "/tmp/kopiaconfig_tier2_rw_*.kopia-password" // archiveConfigPath is the config file the klio-plugin sidecar uses for its own // cluster's backup/WAL-archive operations, mounted from the ArchiveConfigKey // projection. It carries this cluster's own Tier2RecoveryEnabled setting, so it @@ -154,31 +151,25 @@ func (f *Tier2RetentionFeature) Setup() types.StepFunc { // Run executes the tier2 retention feature test. // -// This test validates the complete tier2 retention pipeline using a four-level +// This test validates the complete tier2 retention pipeline using a three-level // verification strategy: // -// 1. Result Verification: Verifies that tier2 contains exactly `keepLatest` backups -// by counting Kopia snapshots. This confirms retention was applied but doesn't -// prove the mechanism is working (could be coincidence or manual deletion). +// 1. Retention Verification: creates more backups than `keepLatest` and verifies +// that tier2 ends up with exactly `keepLatest` backups and that the oldest one +// was the backup the retention manager deleted (the newest survive). This +// exercises the full path: PluginConfiguration CR -> operator -> klio-plugin +// config -> CloseBackup GRPC -> NATS queue -> backup consumer -> Klio-managed +// retention. // -// 2. Mechanism Verification: Queries Kopia directly via `kopia policy list` to verify -// the retention policy was actually configured with the correct `keepLatest` value. -// This proves the full policy propagation path is working: -// PluginConfiguration CR -> operator -> klio-plugin config -> CloseBackup GRPC -> -// NATS queue -> backup consumer -> SetKopiaPolicy() -> ApplyKopiaPolicy() -// -// 3. WAL Retention Verification: Monitors WAL directory count before and after +// 2. WAL Retention Verification: Monitors WAL directory count before and after // retention to verify WAL cleanup is occurring. This is a soft check (logs // warnings only) because WAL retention depends on backup metadata (StartWAL) // and timing, making strict assertions fragile. // -// 4. Tier2 Recovery Gate Verification: this scenario configures tier2 for backup +// 3. Tier2 Recovery Gate Verification: this scenario configures tier2 for backup // only (EnableTier2Recovery: false). Runs an actual `klio restore` as this // cluster's own client identity and verifies the gate log fires, confirming // tier2 was dropped as a recovery source rather than silently used. -// -// The test creates more backups than `keepLatest` to trigger retention, then verifies -// all four levels pass. func (f *Tier2RetentionFeature) Run() types.StepFunc { return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { t.Helper() @@ -187,9 +178,13 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { r, err := resources.New(cfg.Client().RESTConfig()) require.NoError(t, err, "failed to create resources client") - // Track WAL directory count to verify WAL retention (Level 3 verification) + // Track WAL directory count to verify WAL retention (Level 2 verification) var walDirCountAfterFirstBackup int + // Name of the first (oldest) backup that reached tier2. Retention must + // eventually delete it once newer backups push it outside the policy. + var oldestTier2BackupName string + // Create more backups than keepLatest to trigger retention for i, backup := range f.backups { t.Logf("Creating backup %d/%d: %s", i+1, len(f.backups), backup.Name) @@ -219,7 +214,14 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { t.Logf("Tier2 has expected %d backup(s) after backup %d", expectedBackups, i+1) // After the first backup, record the WAL directory count as baseline + // and remember which backup reached tier2 first (the oldest one). if i == 0 { + names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + require.NoError(t, listErr, "failed to list tier2 backups after the first backup") + require.Len(t, names, 1, "expected exactly one tier2 backup after the first backup") + oldestTier2BackupName = names[0] + t.Logf("Oldest tier2 backup recorded: %s", oldestTier2BackupName) + walDirCountAfterFirstBackup, err = countTier2WALDirectories( ctx, r, f.namespace, f.klioServer.Name, f.s3Prefix, f.clusterName) if err != nil { @@ -231,37 +233,27 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { } // ========================================== - // Level 1: Result Verification + // Level 1: Retention Verification // ========================================== - // Verify that tier2 contains exactly keepLatest backups by querying - // the Klio admin API and counting backups with the tier2 annotation. - t.Logf("[Level 1] Result verification: tier2 should have exactly %d backup(s)", f.keepLatest) + // Verify that tier2 contains exactly keepLatest backups and that the + // oldest one was deleted by the retention manager (the newest survive). + t.Logf("[Level 1] Retention verification: tier2 should have exactly %d backup(s)", f.keepLatest) err = wait.For( checkTier2HasBackups(r, f.namespace, f.klioServer.Name, f.keepLatest), wait.WithTimeout(f.replicationTimeout), wait.WithInterval(f.checkInterval), ) require.NoError(t, err, "Level 1 failed: tier2 backup count verification failed") - t.Logf("[Level 1] PASSED: tier2 has exactly %d backup(s)", f.keepLatest) - // ========================================== - // Level 2: Mechanism Verification - // ========================================== - // Query Kopia directly to verify the retention policy was actually set. - // This proves the full propagation path is working, not just the result. - // We use `kopia policy list` to find all policies and check for one - // matching our cluster (hostname) with the expected keepLatest value. - t.Log("[Level 2] Mechanism verification: checking Kopia retention policy is set...") - policyKeepLatest, err := verifyTier2RetentionPolicySet( - ctx, r, f.namespace, f.klioServer.Name, f.clusterName) - require.NoError(t, err, "Level 2 failed: could not verify tier2 retention policy") - require.Equal(t, f.keepLatest, policyKeepLatest, - "Level 2 failed: tier2 Kopia retention policy keepLatest=%d, expected=%d", - policyKeepLatest, f.keepLatest) - t.Logf("[Level 2] PASSED: Kopia retention policy has keepLatest=%d", policyKeepLatest) + survivingNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + require.NoError(t, err, "Level 1 failed: could not list surviving tier2 backups") + require.NotContains(t, survivingNames, oldestTier2BackupName, + "Level 1 failed: the oldest backup should have been deleted by the retention manager") + t.Logf("[Level 1] PASSED: tier2 has exactly %d backup(s) and the oldest (%s) was deleted", + f.keepLatest, oldestTier2BackupName) // ========================================== - // Level 3: WAL Retention Verification (Soft Check) + // Level 2: WAL Retention Verification (Soft Check) // ========================================== // Monitor WAL directory count to verify cleanup is occurring. // This is a soft check (warnings only) because: @@ -271,15 +263,15 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { verifyWALRetention(ctx, t, r, f, walDirCountAfterFirstBackup) // ========================================== - // Level 4: Tier2 Recovery Gate Verification + // Level 3: Tier2 Recovery Gate Verification // ========================================== // This scenario configures tier2 for backup only (EnableTier2Recovery: // false). Run an actual `klio restore` as this cluster's own client // identity and verify the gate log fires, confirming tier2 was dropped // as a recovery source rather than silently used. - t.Log("[Level 4] Tier2 recovery gate verification: klio restore must not use a backup-only tier2...") + t.Log("[Level 3] Tier2 recovery gate verification: klio restore must not use a backup-only tier2...") verifyTier2RecoveryGate(ctx, t, r, f.namespace, f.backups[len(f.backups)-1]) - t.Log("[Level 4] PASSED: klio restore logged that tier2 recovery is disabled") + t.Log("[Level 3] PASSED: klio restore logged that tier2 recovery is disabled") t.Log("Tier2 retention test completed: all verification levels passed") @@ -338,33 +330,33 @@ func verifyWALRetention( t.Helper() if baselineCount == 0 { - t.Log("[Level 3] SKIPPED: no baseline WAL count available") + t.Log("[Level 2] SKIPPED: no baseline WAL count available") return } - t.Log("[Level 3] WAL retention verification: checking WAL directory growth...") + t.Log("[Level 2] WAL retention verification: checking WAL directory growth...") finalCount, err := countTier2WALDirectories( ctx, r, f.namespace, f.klioServer.Name, f.s3Prefix, f.clusterName) if err != nil { - t.Logf("[Level 3] WARNING: could not count final WAL directories: %v", err) + t.Logf("[Level 2] WARNING: could not count final WAL directories: %v", err) return } - t.Logf("[Level 3] WAL directories: %d (was %d after first backup)", finalCount, baselineCount) + t.Logf("[Level 2] WAL directories: %d (was %d after first backup)", finalCount, baselineCount) // WAL retention should prevent unbounded growth. We allow 3x growth // to account for WALs generated during test execution. const maxGrowthFactor = 3 if finalCount > baselineCount*maxGrowthFactor { - t.Logf("[Level 3] WARNING: WAL directory count grew significantly (%d -> %d), "+ + t.Logf("[Level 2] WARNING: WAL directory count grew significantly (%d -> %d), "+ "WAL retention may not be working as expected", baselineCount, finalCount) return } - t.Log("[Level 3] PASSED: WAL directory growth is within acceptable bounds") + t.Log("[Level 2] PASSED: WAL directory growth is within acceptable bounds") } // checkTier2HasBackups checks if tier2 has exactly the expected number of backups. @@ -414,112 +406,46 @@ func checkTier2HasBackups( } } -// verifyTier2RetentionPolicySet verifies that the Kopia retention policy is set in tier2. -// It uses `kopia policy list` to find all policies and searches for one matching the -// cluster name (hostname). This is necessary because policies are stored with -// "username@hostname" format and we don't know the username in the test context. -// Returns the keepLatest value from the policy, or an error if the policy is not set. -// -//nolint:cyclop -func verifyTier2RetentionPolicySet( +// listTier2BackupNames returns the names of the backups currently present in +// tier2 (those carrying the tier2 annotation), ordered newest first by their +// start time. +func listTier2BackupNames( ctx context.Context, r *resources.Resources, namespace string, serverName string, - clusterName string, -) (int, error) { +) ([]string, error) { podName := serverName + klioPodSuffix - // Find the tier2 config file var stdout, stderr bytes.Buffer - findCmd := []string{ - "sh", "-c", - "ls " + tier2KopiaConfigPattern + " 2>/dev/null", + klioCmd := []string{"klio", "admin", "list-backups"} + if err := r.ExecInPod(ctx, namespace, podName, serverContainerName, klioCmd, &stdout, &stderr); err != nil { + return nil, fmt.Errorf("failed to list backups: %w; stderr: %s", err, stderr.String()) } - err := r.ExecInPod(ctx, namespace, podName, serverContainerName, findCmd, &stdout, &stderr) - if err != nil { - return 0, fmt.Errorf("could not find kopia tier2 config: %w", err) + type backupMetadata struct { + Name string `json:"name"` + StartedAt int64 `json:"startedAt"` + Annotations map[string]string `json:"annotations,omitempty"` } - passwordFile := strings.TrimSpace(stdout.String()) - if passwordFile == "" { - return 0, errors.New("kopia tier2 config password file not found") + var backups []backupMetadata + if err := json.Unmarshal(stdout.Bytes(), &backups); err != nil { + return nil, fmt.Errorf("failed to parse backup list: %w", err) } - configFile := strings.TrimSuffix(passwordFile, ".kopia-password") - - // First, use `kopia policy list` to find the full target (username@hostname). - // We can't use `kopia policy show ` directly because Kopia stores policies - // with "username@hostname" format and we don't know the username. - stdout.Reset() - stderr.Reset() - listCmd := []string{ - "kopia", "policy", "list", - "--disable-file-logging", - "--config-file=" + configFile, - "--json", - } + slices.SortFunc(backups, func(a, b backupMetadata) int { + return cmp.Compare(b.StartedAt, a.StartedAt) + }) - err = r.ExecInPod(ctx, namespace, podName, serverContainerName, listCmd, &stdout, &stderr) - if err != nil { - return 0, fmt.Errorf("failed to list kopia policies: %w; stderr: %s", err, stderr.String()) - } - - // Parse to find the target string for our cluster - var policies []struct { - Target struct { - Host string `json:"host"` - User string `json:"userName"` - } `json:"target"` - } - if err := json.Unmarshal(stdout.Bytes(), &policies); err != nil { - return 0, fmt.Errorf("failed to parse kopia policy list output: %w", err) - } - - // Find the full target for our cluster - var targetStr string - for _, p := range policies { - if p.Target.Host == clusterName { - targetStr = p.Target.User + "@" + p.Target.Host - break + names := make([]string, 0, len(backups)) + for i := range backups { + if backups[i].Annotations[tier2AnnotationName] == presentAnnotationValue { + names = append(names, backups[i].Name) } } - if targetStr == "" { - return 0, fmt.Errorf("no policy found for host %q in %d policies", clusterName, len(policies)) - } - - // Now use `kopia policy show` to get the full policy details - stdout.Reset() - stderr.Reset() - showCmd := []string{ - "kopia", "policy", "show", - targetStr, - "--disable-file-logging", - "--config-file=" + configFile, - "--json", - } - - err = r.ExecInPod(ctx, namespace, podName, serverContainerName, showCmd, &stdout, &stderr) - if err != nil { - return 0, fmt.Errorf("failed to show kopia policy for %q: %w; stderr: %s", targetStr, err, stderr.String()) - } - - // Parse the policy details - var policyDetail struct { - RetentionPolicy struct { - KeepLatest *int `json:"keepLatest"` - } `json:"retention"` - } - if err := json.Unmarshal(stdout.Bytes(), &policyDetail); err != nil { - return 0, fmt.Errorf("failed to parse kopia policy show output: %w", err) - } - - if policyDetail.RetentionPolicy.KeepLatest == nil { - return 0, fmt.Errorf("policy found for target %q but keepLatest not set", targetStr) - } - return *policyDetail.RetentionPolicy.KeepLatest, nil + return names, nil } // countTier2WALDirectories counts the number of WAL prefix directories in tier2 S3 storage. From 97ed586547c3d07e00268b7ea289bb48b8687904 Mon Sep 17 00:00:00 2001 From: Gabriele Quaresima Date: Mon, 7 Sep 2026 17:35:15 +0200 Subject: [PATCH 02/23] feat(retention): apply retention on demand with a new command Add a klio retention apply command and an ApplyRetention gRPC call that apply the configured retention immediately, without waiting for the next backup. This frees space on demand, for example after shrinking the policy on a nearly-full disk. The command reads the client configuration and sends the tier1 and tier2 policies to the server, which enqueues a maintenance-only task. The backup consumer then runs the same per-tier retention path used after a backup, so there is a single retention implementation. Assisted-by: Claude Signed-off-by: Gabriele Quaresima --- core/cmd/retention/apply.go | 105 ++++++++++++ core/cmd/retention/retention.go | 38 +++++ core/cmd/root.go | 2 + core/internal/consumer/backup.go | 28 ++++ core/internal/consumer/processbackup_test.go | 48 ++++-- core/internal/grpc/klio_wal.pb.go | 152 ++++++++++++++++-- core/internal/grpc/klio_wal_grpc.pb.go | 38 +++++ core/internal/queue/backup.go | 5 + core/internal/server/walserver/backup.go | 26 +++ core/proto/klio_wal.proto | 23 +++ documentation/.wordlist.txt | 3 + documentation/web/docs/developer/_protocol.md | 36 +++++ documentation/web/docs/user/cli/klio.md | 1 + .../web/docs/user/cli/klio_retention.md | 37 +++++ .../web/docs/user/cli/klio_retention_apply.md | 44 +++++ .../web/docs/user/managing_storage.md | 3 +- .../web/docs/user/plugin_configuration.md | 4 + operator/test/e2e/tier2_retention_test.go | 19 +-- .../test/klio/features/tier2_retention.go | 146 ++++++++++++++--- 19 files changed, 699 insertions(+), 59 deletions(-) create mode 100644 core/cmd/retention/apply.go create mode 100644 core/cmd/retention/retention.go create mode 100644 documentation/web/docs/user/cli/klio_retention.md create mode 100644 documentation/web/docs/user/cli/klio_retention_apply.md diff --git a/core/cmd/retention/apply.go b/core/cmd/retention/apply.go new file mode 100644 index 00000000..94f0ca39 --- /dev/null +++ b/core/cmd/retention/apply.go @@ -0,0 +1,105 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package retention + +import ( + "fmt" + + "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudnative-pg/klio/core/internal/backupfailure" + "github.com/cloudnative-pg/klio/core/internal/cli" + "github.com/cloudnative-pg/klio/core/internal/client/klioclient/grpcclient" + "github.com/cloudnative-pg/klio/core/internal/grpc" + "github.com/cloudnative-pg/klio/core/pkg/config" +) + +// applyCmd represents the `retention apply` command. +// +//nolint:gochecknoglobals +var applyCmd = &cobra.Command{ + Use: "apply", + Short: "Apply the configured retention policy immediately", + Long: "Apply the retention policy from the configuration to the target cluster " + + "without waiting for the next backup, to free space on demand.", + RunE: cli.RunEWithExitCode(runApply), +} + +func runApply(cmd *cobra.Command, _ []string) error { + contextLogger := log.FromContext(cmd.Context()) + + var configuration config.Data + + // IMPORTANT: this requires this program to be built with "-tags viper_bind_struct" + // when using environment variables + if err := viper.Unmarshal(&configuration); err != nil { + return fmt.Errorf("could not unmarshal configuration: %w", err) + } + + // Sets the default values, to be overridden by the user configuration. + configuration.SetDefaults() + + if configuration.Client == (config.ClientConfig{}) { + return cli.ErrClientSectionIsRequired + } + if configuration.Client.Wal == (config.WalRepositoryClientConfig{}) { + return cli.ErrKlioClientSectionIsRequired + } + + if err := configuration.Validate(); err != nil { + return fmt.Errorf("configuration validation error: %w", err) + } + + tier1RetentionPolicy, err := configuration.Tier1RetentionPolicy.MarshalWire() + if err != nil { + return fmt.Errorf("while serializing the tier1 retention policy: %w", err) + } + + tier2RetentionPolicy, err := configuration.Tier2RetentionPolicy.MarshalWire() + if err != nil { + return fmt.Errorf("while serializing the tier2 retention policy: %w", err) + } + + grpcClient, err := grpcclient.Connect(&configuration.Client, configuration.Client.Wal.Address) + if err != nil { + return cli.NewCodedError( + fmt.Errorf("while connecting to the Klio server: %w", err), + backupfailure.RepositoryError.ExitCode) + } + + result, err := grpcClient.ApplyRetention(cmd.Context(), &grpc.ApplyRetentionRequest{ + ClusterName: configuration.Client.ClusterName, + Tier1RetentionPolicy: tier1RetentionPolicy, + Tier2RetentionPolicy: tier2RetentionPolicy, + }) + if err != nil { + return cli.NewCodedError( + fmt.Errorf("while applying retention: %w", err), + backupfailure.RepositoryError.ExitCode) + } + + if result.GetScheduled() { + contextLogger.Info("Retention apply scheduled", "cluster", configuration.Client.ClusterName) + } + + return nil +} diff --git a/core/cmd/retention/retention.go b/core/cmd/retention/retention.go new file mode 100644 index 00000000..c0a6d07a --- /dev/null +++ b/core/cmd/retention/retention.go @@ -0,0 +1,38 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package retention implements the `klio retention` command group. +package retention + +import ( + "github.com/spf13/cobra" +) + +// RetentionCmd is the parent command for retention operations. +// +//nolint:gochecknoglobals +var RetentionCmd = &cobra.Command{ + Use: "retention", + Short: "Manage the retention policy", +} + +//nolint:gochecknoinits +func init() { + RetentionCmd.AddCommand(applyCmd) +} diff --git a/core/cmd/root.go b/core/cmd/root.go index da337ba0..15af469c 100644 --- a/core/cmd/root.go +++ b/core/cmd/root.go @@ -33,6 +33,7 @@ import ( "github.com/cloudnative-pg/klio/core/cmd/admin" "github.com/cloudnative-pg/klio/core/cmd/backup" "github.com/cloudnative-pg/klio/core/cmd/cnpgi" + "github.com/cloudnative-pg/klio/core/cmd/retention" "github.com/cloudnative-pg/klio/core/cmd/server" "github.com/cloudnative-pg/klio/core/cmd/walplayer" @@ -122,6 +123,7 @@ func init() { rootCmd.AddCommand(backup.BackupCmd) rootCmd.AddCommand(walplayer.WalPlayerCmd) rootCmd.AddCommand(cnpgi.CnpgiCmd) + rootCmd.AddCommand(retention.RetentionCmd) rootCmd.AddCommand(admin.AdminCmd) } diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index fefe4a13..7383bd46 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -165,6 +165,12 @@ func (d *Backup) processBackup(ctx context.Context, task *queue.BackupTask) erro contextLogger := log.FromContext(ctx) contextLogger.Info("Processing backup", "task", task) + // A maintenance-only task applies retention to the cluster without a new + // backup, so it skips snapshot listing, verification and the tier2 relay. + if task.MaintenanceOnly { + return d.maintainOnly(ctx, task) + } + entries, err := d.steps.listManifests(ctx, task.ClusterName) if err != nil { return err @@ -232,6 +238,28 @@ func (d *Backup) relayAndMaintain(ctx context.Context, task *queue.BackupTask, e return nil } +// maintainOnly applies the per-tier retention policies to a cluster without a +// new backup, driven by a maintenance-only task. tier2 maintenance runs first +// (when tier2 is configured) so the tier1 guard sees an up-to-date tier2 +// catalog before tier1 deletes anything; a tier2 failure is fatal so the task +// is retried, while tier1 maintenance is best-effort. +func (d *Backup) maintainOnly(ctx context.Context, task *queue.BackupTask) error { + contextLogger := log.FromContext(ctx) + contextLogger.Info("Applying on-demand retention", "cluster", task.ClusterName) + + if d.tier2Enabled { + if err := d.steps.maintainTier2(ctx, task); err != nil { + return err + } + } + + if err := d.steps.maintainTier1(ctx, task); err != nil { + contextLogger.Error(err, "Error while applying tier1 maintenance, skipping") + } + + return nil +} + // relayTier2 migrates the cluster's backups to tier2 and verifies them there. // tier2 retention/WAL cleanup is handled separately by maintainTier2. func (d *Backup) relayTier2(ctx context.Context, task *queue.BackupTask, entries []kopia.Manifest) error { diff --git a/core/internal/consumer/processbackup_test.go b/core/internal/consumer/processbackup_test.go index 4557324b..f307f530 100644 --- a/core/internal/consumer/processbackup_test.go +++ b/core/internal/consumer/processbackup_test.go @@ -81,15 +81,16 @@ func TestProcessBackup(t *testing.T) { someEntries := []kopia.Manifest{{}} tests := []struct { - name string - sendToTier2 bool - tier2Enabled bool - manifests []kopia.Manifest - manifestsErr error - verifyErr error - relayErr error - maintain2Err error - maintainErr error + name string + sendToTier2 bool + tier2Enabled bool + maintenanceOnly bool + manifests []kopia.Manifest + manifestsErr error + verifyErr error + relayErr error + maintain2Err error + maintainErr error wantErr bool wantVerify bool @@ -166,6 +167,29 @@ func TestProcessBackup(t *testing.T) { wantVerify: true, wantMaintain: true, }, + { + // The maintenance-only path must skip listing (so a listing error is + // irrelevant), verification and relay, and only maintain tier1. + name: "maintenance-only tier1 skips listing verify and relay", + maintenanceOnly: true, + manifestsErr: errBoom, + wantMaintain: true, + }, + { + name: "maintenance-only with tier2 maintains both tiers", + maintenanceOnly: true, + tier2Enabled: true, + wantMaintain2: true, + wantMaintain: true, + }, + { + name: "maintenance-only tier2 failure is retried before tier1", + maintenanceOnly: true, + tier2Enabled: true, + maintain2Err: errBoom, + wantErr: true, + wantMaintain2: true, + }, } for _, tt := range tests { @@ -184,7 +208,11 @@ func TestProcessBackup(t *testing.T) { steps: stub, } - task := &queue.BackupTask{ClusterName: "cluster", SendToTier2: tt.sendToTier2} + task := &queue.BackupTask{ + ClusterName: "cluster", + SendToTier2: tt.sendToTier2, + MaintenanceOnly: tt.maintenanceOnly, + } err := b.processBackup(context.Background(), task) if (err != nil) != tt.wantErr { diff --git a/core/internal/grpc/klio_wal.pb.go b/core/internal/grpc/klio_wal.pb.go index 7e53e5b8..034ef126 100644 --- a/core/internal/grpc/klio_wal.pb.go +++ b/core/internal/grpc/klio_wal.pb.go @@ -894,6 +894,119 @@ func (x *CloseBackupResult) GetMissingWalFiles() []string { return nil } +// This is sent to the WAL server to apply a retention policy to a cluster +// immediately, without waiting for the next backup. +type ApplyRetentionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The name of the cluster whose backups should be pruned. + ClusterName string `protobuf:"bytes,1,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"` + // The tier1 retention policy to apply, as a JSON-serialized policy. When + // empty, tier1 keeps every backup. + Tier1RetentionPolicy string `protobuf:"bytes,2,opt,name=tier1_retention_policy,json=tier1RetentionPolicy,proto3" json:"tier1_retention_policy,omitempty"` + // The tier2 retention policy to apply, as a JSON-serialized policy. When + // empty, tier2 keeps every backup. + Tier2RetentionPolicy string `protobuf:"bytes,3,opt,name=tier2_retention_policy,json=tier2RetentionPolicy,proto3" json:"tier2_retention_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ApplyRetentionRequest) Reset() { + *x = ApplyRetentionRequest{} + mi := &file_proto_klio_wal_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ApplyRetentionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ApplyRetentionRequest) ProtoMessage() {} + +func (x *ApplyRetentionRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_klio_wal_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ApplyRetentionRequest.ProtoReflect.Descriptor instead. +func (*ApplyRetentionRequest) Descriptor() ([]byte, []int) { + return file_proto_klio_wal_proto_rawDescGZIP(), []int{14} +} + +func (x *ApplyRetentionRequest) GetClusterName() string { + if x != nil { + return x.ClusterName + } + return "" +} + +func (x *ApplyRetentionRequest) GetTier1RetentionPolicy() string { + if x != nil { + return x.Tier1RetentionPolicy + } + return "" +} + +func (x *ApplyRetentionRequest) GetTier2RetentionPolicy() string { + if x != nil { + return x.Tier2RetentionPolicy + } + return "" +} + +// This is sent by the WAL server in response to an ApplyRetentionRequest. +type ApplyRetentionResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // True when the retention has been scheduled for execution. + Scheduled bool `protobuf:"varint,1,opt,name=scheduled,proto3" json:"scheduled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ApplyRetentionResult) Reset() { + *x = ApplyRetentionResult{} + mi := &file_proto_klio_wal_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ApplyRetentionResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ApplyRetentionResult) ProtoMessage() {} + +func (x *ApplyRetentionResult) ProtoReflect() protoreflect.Message { + mi := &file_proto_klio_wal_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ApplyRetentionResult.ProtoReflect.Descriptor instead. +func (*ApplyRetentionResult) Descriptor() ([]byte, []int) { + return file_proto_klio_wal_proto_rawDescGZIP(), []int{15} +} + +func (x *ApplyRetentionResult) GetScheduled() bool { + if x != nil { + return x.Scheduled + } + return false +} + var File_proto_klio_wal_proto protoreflect.FileDescriptor const file_proto_klio_wal_proto_rawDesc = "" + @@ -955,14 +1068,21 @@ const file_proto_klio_wal_proto_rawDesc = "" + "\x16tier1_retention_policy\x18\v \x01(\tR\x14tier1RetentionPolicy\"f\n" + "\x11CloseBackupResult\x12%\n" + "\x0etier2_schedule\x18\x01 \x01(\bR\rtier2Schedule\x12*\n" + - "\x11missing_wal_files\x18\x02 \x03(\tR\x0fmissingWalFiles2\xd8\x03\n" + + "\x11missing_wal_files\x18\x02 \x03(\tR\x0fmissingWalFiles\"\xa6\x01\n" + + "\x15ApplyRetentionRequest\x12!\n" + + "\fcluster_name\x18\x01 \x01(\tR\vclusterName\x124\n" + + "\x16tier1_retention_policy\x18\x02 \x01(\tR\x14tier1RetentionPolicy\x124\n" + + "\x16tier2_retention_policy\x18\x03 \x01(\tR\x14tier2RetentionPolicy\"4\n" + + "\x14ApplyRetentionResult\x12\x1c\n" + + "\tscheduled\x18\x01 \x01(\bR\tscheduled2\xb3\x04\n" + "\x03WAL\x12:\n" + "\x03Put\x12\x17.klio.wal.v1.PutRequest\x1a\x16.klio.wal.v1.PutResult\"\x00(\x01\x12:\n" + "\x03Get\x12\x17.klio.wal.v1.GetRequest\x1a\x16.klio.wal.v1.GetResult\"\x000\x01\x12N\n" + "\vGetMetadata\x12\x1f.klio.wal.v1.GetMetadataRequest\x1a\x1c.klio.wal.v1.ClusterMetadata\"\x00\x12\\\n" + "\x0fRequestWALStart\x12#.klio.wal.v1.RequestWALStartRequest\x1a\".klio.wal.v1.RequestWALStartResult\"\x00\x12Y\n" + "\x0eResetWALStream\x12\".klio.wal.v1.ResetWALStreamRequest\x1a!.klio.wal.v1.ResetWALStreamResult\"\x00\x12P\n" + - "\vCloseBackup\x12\x1f.klio.wal.v1.CloseBackupRequest\x1a\x1e.klio.wal.v1.CloseBackupResult\"\x00B3Z1github.com/cloudnative-pg/klio/core/internal/grpcb\x06proto3" + "\vCloseBackup\x12\x1f.klio.wal.v1.CloseBackupRequest\x1a\x1e.klio.wal.v1.CloseBackupResult\"\x00\x12Y\n" + + "\x0eApplyRetention\x12\".klio.wal.v1.ApplyRetentionRequest\x1a!.klio.wal.v1.ApplyRetentionResult\"\x00B3Z1github.com/cloudnative-pg/klio/core/internal/grpcb\x06proto3" var ( file_proto_klio_wal_proto_rawDescOnce sync.Once @@ -976,7 +1096,7 @@ func file_proto_klio_wal_proto_rawDescGZIP() []byte { return file_proto_klio_wal_proto_rawDescData } -var file_proto_klio_wal_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_proto_klio_wal_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_proto_klio_wal_proto_goTypes = []any{ (*PutRequest)(nil), // 0: klio.wal.v1.PutRequest (*PutResult)(nil), // 1: klio.wal.v1.PutResult @@ -992,25 +1112,29 @@ var file_proto_klio_wal_proto_goTypes = []any{ (*StartWALFile)(nil), // 11: klio.wal.v1.StartWALFile (*CloseBackupRequest)(nil), // 12: klio.wal.v1.CloseBackupRequest (*CloseBackupResult)(nil), // 13: klio.wal.v1.CloseBackupResult - (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp + (*ApplyRetentionRequest)(nil), // 14: klio.wal.v1.ApplyRetentionRequest + (*ApplyRetentionResult)(nil), // 15: klio.wal.v1.ApplyRetentionResult + (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp } var file_proto_klio_wal_proto_depIdxs = []int32{ 10, // 0: klio.wal.v1.ClusterMetadata.gaps:type_name -> klio.wal.v1.WALGap - 14, // 1: klio.wal.v1.WALGap.ts:type_name -> google.protobuf.Timestamp + 16, // 1: klio.wal.v1.WALGap.ts:type_name -> google.protobuf.Timestamp 0, // 2: klio.wal.v1.WAL.Put:input_type -> klio.wal.v1.PutRequest 3, // 3: klio.wal.v1.WAL.Get:input_type -> klio.wal.v1.GetRequest 2, // 4: klio.wal.v1.WAL.GetMetadata:input_type -> klio.wal.v1.GetMetadataRequest 5, // 5: klio.wal.v1.WAL.RequestWALStart:input_type -> klio.wal.v1.RequestWALStartRequest 7, // 6: klio.wal.v1.WAL.ResetWALStream:input_type -> klio.wal.v1.ResetWALStreamRequest 12, // 7: klio.wal.v1.WAL.CloseBackup:input_type -> klio.wal.v1.CloseBackupRequest - 1, // 8: klio.wal.v1.WAL.Put:output_type -> klio.wal.v1.PutResult - 4, // 9: klio.wal.v1.WAL.Get:output_type -> klio.wal.v1.GetResult - 9, // 10: klio.wal.v1.WAL.GetMetadata:output_type -> klio.wal.v1.ClusterMetadata - 6, // 11: klio.wal.v1.WAL.RequestWALStart:output_type -> klio.wal.v1.RequestWALStartResult - 8, // 12: klio.wal.v1.WAL.ResetWALStream:output_type -> klio.wal.v1.ResetWALStreamResult - 13, // 13: klio.wal.v1.WAL.CloseBackup:output_type -> klio.wal.v1.CloseBackupResult - 8, // [8:14] is the sub-list for method output_type - 2, // [2:8] is the sub-list for method input_type + 14, // 8: klio.wal.v1.WAL.ApplyRetention:input_type -> klio.wal.v1.ApplyRetentionRequest + 1, // 9: klio.wal.v1.WAL.Put:output_type -> klio.wal.v1.PutResult + 4, // 10: klio.wal.v1.WAL.Get:output_type -> klio.wal.v1.GetResult + 9, // 11: klio.wal.v1.WAL.GetMetadata:output_type -> klio.wal.v1.ClusterMetadata + 6, // 12: klio.wal.v1.WAL.RequestWALStart:output_type -> klio.wal.v1.RequestWALStartResult + 8, // 13: klio.wal.v1.WAL.ResetWALStream:output_type -> klio.wal.v1.ResetWALStreamResult + 13, // 14: klio.wal.v1.WAL.CloseBackup:output_type -> klio.wal.v1.CloseBackupResult + 15, // 15: klio.wal.v1.WAL.ApplyRetention:output_type -> klio.wal.v1.ApplyRetentionResult + 9, // [9:16] is the sub-list for method output_type + 2, // [2:9] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name @@ -1027,7 +1151,7 @@ func file_proto_klio_wal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_klio_wal_proto_rawDesc), len(file_proto_klio_wal_proto_rawDesc)), NumEnums: 0, - NumMessages: 14, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, diff --git a/core/internal/grpc/klio_wal_grpc.pb.go b/core/internal/grpc/klio_wal_grpc.pb.go index c78a82b5..fef0bfd8 100644 --- a/core/internal/grpc/klio_wal_grpc.pb.go +++ b/core/internal/grpc/klio_wal_grpc.pb.go @@ -44,6 +44,7 @@ const ( WAL_RequestWALStart_FullMethodName = "/klio.wal.v1.WAL/RequestWALStart" WAL_ResetWALStream_FullMethodName = "/klio.wal.v1.WAL/ResetWALStream" WAL_CloseBackup_FullMethodName = "/klio.wal.v1.WAL/CloseBackup" + WAL_ApplyRetention_FullMethodName = "/klio.wal.v1.WAL/ApplyRetention" ) // WALClient is the client API for WAL service. @@ -56,6 +57,7 @@ type WALClient interface { RequestWALStart(ctx context.Context, in *RequestWALStartRequest, opts ...grpc.CallOption) (*RequestWALStartResult, error) ResetWALStream(ctx context.Context, in *ResetWALStreamRequest, opts ...grpc.CallOption) (*ResetWALStreamResult, error) CloseBackup(ctx context.Context, in *CloseBackupRequest, opts ...grpc.CallOption) (*CloseBackupResult, error) + ApplyRetention(ctx context.Context, in *ApplyRetentionRequest, opts ...grpc.CallOption) (*ApplyRetentionResult, error) } type wALClient struct { @@ -138,6 +140,16 @@ func (c *wALClient) CloseBackup(ctx context.Context, in *CloseBackupRequest, opt return out, nil } +func (c *wALClient) ApplyRetention(ctx context.Context, in *ApplyRetentionRequest, opts ...grpc.CallOption) (*ApplyRetentionResult, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ApplyRetentionResult) + err := c.cc.Invoke(ctx, WAL_ApplyRetention_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // WALServer is the server API for WAL service. // All implementations must embed UnimplementedWALServer // for forward compatibility. @@ -148,6 +160,7 @@ type WALServer interface { RequestWALStart(context.Context, *RequestWALStartRequest) (*RequestWALStartResult, error) ResetWALStream(context.Context, *ResetWALStreamRequest) (*ResetWALStreamResult, error) CloseBackup(context.Context, *CloseBackupRequest) (*CloseBackupResult, error) + ApplyRetention(context.Context, *ApplyRetentionRequest) (*ApplyRetentionResult, error) mustEmbedUnimplementedWALServer() } @@ -176,6 +189,9 @@ func (UnimplementedWALServer) ResetWALStream(context.Context, *ResetWALStreamReq func (UnimplementedWALServer) CloseBackup(context.Context, *CloseBackupRequest) (*CloseBackupResult, error) { return nil, status.Error(codes.Unimplemented, "method CloseBackup not implemented") } +func (UnimplementedWALServer) ApplyRetention(context.Context, *ApplyRetentionRequest) (*ApplyRetentionResult, error) { + return nil, status.Error(codes.Unimplemented, "method ApplyRetention not implemented") +} func (UnimplementedWALServer) mustEmbedUnimplementedWALServer() {} func (UnimplementedWALServer) testEmbeddedByValue() {} @@ -287,6 +303,24 @@ func _WAL_CloseBackup_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _WAL_ApplyRetention_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ApplyRetentionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WALServer).ApplyRetention(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WAL_ApplyRetention_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WALServer).ApplyRetention(ctx, req.(*ApplyRetentionRequest)) + } + return interceptor(ctx, in, info, handler) +} + // WAL_ServiceDesc is the grpc.ServiceDesc for WAL service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -310,6 +344,10 @@ var WAL_ServiceDesc = grpc.ServiceDesc{ MethodName: "CloseBackup", Handler: _WAL_CloseBackup_Handler, }, + { + MethodName: "ApplyRetention", + Handler: _WAL_ApplyRetention_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/core/internal/queue/backup.go b/core/internal/queue/backup.go index fb08697b..b91fb0fa 100644 --- a/core/internal/queue/backup.go +++ b/core/internal/queue/backup.go @@ -40,6 +40,11 @@ type BackupTask struct { // and maintenance) without touching tier2. SendToTier2 bool `json:"sendToTier2,omitempty"` + // MaintenanceOnly requests the consumer to apply retention to the cluster + // without a new backup: no snapshot listing, verification or tier2 relay is + // performed, only the per-tier retention and WAL cleanup. + MaintenanceOnly bool `json:"maintenanceOnly,omitempty"` + // Tier1RetentionPolicy is the retention policy to apply to tier1. Its zero // value keeps every backup. Tier1RetentionPolicy retention.Policy `json:"tier1RetentionPolicy,omitzero"` diff --git a/core/internal/server/walserver/backup.go b/core/internal/server/walserver/backup.go index a47fbcca..e200b67b 100644 --- a/core/internal/server/walserver/backup.go +++ b/core/internal/server/walserver/backup.go @@ -72,6 +72,32 @@ func (w *Implementation) CloseBackup( }, nil } +// ApplyRetention implements the ApplyRetention GRPC call. It enqueues a +// maintenance-only task so the backup consumer applies the retention policies +// to the cluster immediately, without waiting for the next backup. +func (w *Implementation) ApplyRetention( + ctx context.Context, + request *grpc.ApplyRetentionRequest, +) (*grpc.ApplyRetentionResult, error) { + if w.queue == nil { + return nil, status.Errorf(codes.Internal, "queue service is uninitialized") + } + + tier1Policy := parseRetentionPolicy(ctx, "tier1", request.GetTier1RetentionPolicy()) + tier2Policy := parseRetentionPolicy(ctx, "tier2", request.GetTier2RetentionPolicy()) + + if err := w.queue.NotifyBackupReceived(ctx, &queue.BackupTask{ + ClusterName: request.GetClusterName(), + MaintenanceOnly: true, + Tier1RetentionPolicy: tier1Policy, + Tier2RetentionPolicy: tier2Policy, + }); err != nil { + return nil, status.Errorf(codes.Internal, "while scheduling retention: %v", err) + } + + return &grpc.ApplyRetentionResult{Scheduled: true}, nil +} + func (w *Implementation) scheduleBackupRelay(ctx context.Context, request *grpc.CloseBackupRequest) error { if w.queue == nil { return status.Errorf( diff --git a/core/proto/klio_wal.proto b/core/proto/klio_wal.proto index 02840971..c66b1dbe 100644 --- a/core/proto/klio_wal.proto +++ b/core/proto/klio_wal.proto @@ -33,6 +33,8 @@ service WAL { rpc ResetWALStream(ResetWALStreamRequest) returns (ResetWALStreamResult) {} rpc CloseBackup(CloseBackupRequest) returns (CloseBackupResult) {} + + rpc ApplyRetention(ApplyRetentionRequest) returns (ApplyRetentionResult) {} } message PutRequest { @@ -180,3 +182,24 @@ message CloseBackupResult { // uploaded to tier1 repeated string missing_wal_files = 2; } + +// This is sent to the WAL server to apply a retention policy to a cluster +// immediately, without waiting for the next backup. +message ApplyRetentionRequest { + // The name of the cluster whose backups should be pruned. + string cluster_name = 1; + + // The tier1 retention policy to apply, as a JSON-serialized policy. When + // empty, tier1 keeps every backup. + string tier1_retention_policy = 2; + + // The tier2 retention policy to apply, as a JSON-serialized policy. When + // empty, tier2 keeps every backup. + string tier2_retention_policy = 3; +} + +// This is sent by the WAL server in response to an ApplyRetentionRequest. +message ApplyRetentionResult { + // True when the retention has been scheduled for execution. + bool scheduled = 1; +} diff --git a/documentation/.wordlist.txt b/documentation/.wordlist.txt index da8eafa3..40cb4f6a 100644 --- a/documentation/.wordlist.txt +++ b/documentation/.wordlist.txt @@ -1,6 +1,9 @@ ACLs AES APIs +ApplyRetention +ApplyRetentionRequest +ApplyRetentionResult BackupID BackupManifest Benchmarking diff --git a/documentation/web/docs/developer/_protocol.md b/documentation/web/docs/developer/_protocol.md index e6638093..00d0d50f 100644 --- a/documentation/web/docs/developer/_protocol.md +++ b/documentation/web/docs/developer/_protocol.md @@ -27,6 +27,8 @@ - [Admin](#klio-wal-v1-Admin) - [klio_wal.proto](#klio_wal-proto) + - [ApplyRetentionRequest](#klio-wal-v1-ApplyRetentionRequest) + - [ApplyRetentionResult](#klio-wal-v1-ApplyRetentionResult) - [CloseBackupRequest](#klio-wal-v1-CloseBackupRequest) - [CloseBackupResult](#klio-wal-v1-CloseBackupResult) - [ClusterMetadata](#klio-wal-v1-ClusterMetadata) @@ -334,6 +336,39 @@ Tier represents a storage tier in the backup system. + + +### ApplyRetentionRequest +This is sent to the WAL server to apply a retention policy to a cluster +immediately, without waiting for the next backup. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| cluster_name | [string](#string) | | The name of the cluster whose backups should be pruned. | +| tier1_retention_policy | [string](#string) | | The tier1 retention policy to apply, as a JSON-serialized policy. When empty, tier1 keeps every backup. | +| tier2_retention_policy | [string](#string) | | The tier2 retention policy to apply, as a JSON-serialized policy. When empty, tier2 keeps every backup. | + + + + + + + + +### ApplyRetentionResult +This is sent by the WAL server in response to an ApplyRetentionRequest. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| scheduled | [bool](#bool) | | True when the retention has been scheduled for execution. | + + + + + + ### CloseBackupRequest @@ -593,6 +628,7 @@ feature. | RequestWALStart | [RequestWALStartRequest](#klio-wal-v1-RequestWALStartRequest) | [RequestWALStartResult](#klio-wal-v1-RequestWALStartResult) | | | ResetWALStream | [ResetWALStreamRequest](#klio-wal-v1-ResetWALStreamRequest) | [ResetWALStreamResult](#klio-wal-v1-ResetWALStreamResult) | | | CloseBackup | [CloseBackupRequest](#klio-wal-v1-CloseBackupRequest) | [CloseBackupResult](#klio-wal-v1-CloseBackupResult) | | +| ApplyRetention | [ApplyRetentionRequest](#klio-wal-v1-ApplyRetentionRequest) | [ApplyRetentionResult](#klio-wal-v1-ApplyRetentionResult) | | diff --git a/documentation/web/docs/user/cli/klio.md b/documentation/web/docs/user/cli/klio.md index 6febaf61..a8372388 100644 --- a/documentation/web/docs/user/cli/klio.md +++ b/documentation/web/docs/user/cli/klio.md @@ -43,6 +43,7 @@ run directly by end users. * [klio get-wal](klio_get-wal.md) - Get a WAL from the target Klio server * [klio reset-lsn](klio_reset-lsn.md) - Reset the replication status to the latest flush LSN * [klio restore](klio_restore.md) - Restore a PostgreSQL cluster from a Klio server +* [klio retention](klio_retention.md) - Manage the retention policy * [klio send-wal](klio_send-wal.md) - Upload the cluster's WALs to the target Klio server * [klio server](klio_server.md) - Starts and manage a Klio server * [klio wal-player](klio_wal-player.md) - WAL Player Commands diff --git a/documentation/web/docs/user/cli/klio_retention.md b/documentation/web/docs/user/cli/klio_retention.md new file mode 100644 index 00000000..f1f90607 --- /dev/null +++ b/documentation/web/docs/user/cli/klio_retention.md @@ -0,0 +1,37 @@ +--- +title: klio retention +--- + +## klio retention + +Manage the retention policy + +### Options + +``` + -h, --help help for retention +``` + +### Options inherited from parent commands + +``` + --config string config file (default is $HOME/.klio.yaml) + --debug enable debug logging + --log-destination string where the log stream will be written + --log-field-level string JSON log field to report severity in (default: level) + --log-field-timestamp string JSON log field to report timestamp in (default: ts) + --log-level string the desired log level, one of error, info, debug and trace (default "info") + --log-truncate-destination truncate the log destination on open instead of appending to it (ignored for FIFOs) + --pprof-server string enable the PPROF server using the specified address + --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) + --zap-encoder encoder Zap log encoding (one of 'json' or 'console') + --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' or any integer value > 0 which corresponds to custom debug levels of increasing verbosity + --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). + --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. +``` + +### SEE ALSO + +* [klio](klio.md) - PostgreSQL Backup & Recovery for CloudNativePG +* [klio retention apply](klio_retention_apply.md) - Apply the configured retention policy immediately + diff --git a/documentation/web/docs/user/cli/klio_retention_apply.md b/documentation/web/docs/user/cli/klio_retention_apply.md new file mode 100644 index 00000000..7f621154 --- /dev/null +++ b/documentation/web/docs/user/cli/klio_retention_apply.md @@ -0,0 +1,44 @@ +--- +title: klio retention apply +--- + +## klio retention apply + +Apply the configured retention policy immediately + +### Synopsis + +Apply the retention policy from the configuration to the target cluster without waiting for the next backup, to free space on demand. + +``` +klio retention apply [flags] +``` + +### Options + +``` + -h, --help help for apply +``` + +### Options inherited from parent commands + +``` + --config string config file (default is $HOME/.klio.yaml) + --debug enable debug logging + --log-destination string where the log stream will be written + --log-field-level string JSON log field to report severity in (default: level) + --log-field-timestamp string JSON log field to report timestamp in (default: ts) + --log-level string the desired log level, one of error, info, debug and trace (default "info") + --log-truncate-destination truncate the log destination on open instead of appending to it (ignored for FIFOs) + --pprof-server string enable the PPROF server using the specified address + --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) + --zap-encoder encoder Zap log encoding (one of 'json' or 'console') + --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' or any integer value > 0 which corresponds to custom debug levels of increasing verbosity + --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). + --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. +``` + +### SEE ALSO + +* [klio retention](klio_retention.md) - Manage the retention policy + diff --git a/documentation/web/docs/user/managing_storage.md b/documentation/web/docs/user/managing_storage.md index ab551307..10ba6616 100644 --- a/documentation/web/docs/user/managing_storage.md +++ b/documentation/web/docs/user/managing_storage.md @@ -275,7 +275,8 @@ and running maintenance manually. 1. **Configure retention policies**: The most effective way to control storage growth is through properly configured retention policies, which automatically delete old backups and WAL files no longer needed for - recovery. See + recovery. After tightening a policy, run `klio retention apply` to enforce + it immediately instead of waiting for the next backup. See [Retention Policies](plugin_configuration.md#retention-policies) for configuration details. diff --git a/documentation/web/docs/user/plugin_configuration.md b/documentation/web/docs/user/plugin_configuration.md index 3f4493cd..e7d1d2b6 100644 --- a/documentation/web/docs/user/plugin_configuration.md +++ b/documentation/web/docs/user/plugin_configuration.md @@ -243,6 +243,10 @@ for recovery by any retained backup. The retention policy is optional and must be set to at least `1` when present. Omit it entirely to keep every backup. +A change to the retention policy takes effect the next time a backup is taken. +To apply it immediately, for example to reclaim space after tightening the +policy, run `klio retention apply`. + ### Compression policies By default, Kopia stores base backup data uncompressed. You can enable diff --git a/operator/test/e2e/tier2_retention_test.go b/operator/test/e2e/tier2_retention_test.go index f93d6f15..1312f120 100644 --- a/operator/test/e2e/tier2_retention_test.go +++ b/operator/test/e2e/tier2_retention_test.go @@ -307,15 +307,16 @@ func NewTier2RetentionFeatureConfig( } return klioFeatures.Tier2RetentionFeatureConfig{ - Name: name, - Setup: scenario.Setup, - Teardown: scenario.Teardown, - Backups: backups, - KlioServer: klioServer, - Namespace: namespace, - KeepLatest: tier2RetentionKeepNum, - ClusterName: cnpgClusterName, - S3Prefix: s3Prefix, + Name: name, + Setup: scenario.Setup, + Teardown: scenario.Teardown, + Backups: backups, + KlioServer: klioServer, + Namespace: namespace, + KeepLatest: tier2RetentionKeepNum, + ClusterName: cnpgClusterName, + S3Prefix: s3Prefix, + PluginConfigurationName: pluginConfigurationName, } } diff --git a/operator/test/klio/features/tier2_retention.go b/operator/test/klio/features/tier2_retention.go index f33a3145..e85609db 100644 --- a/operator/test/klio/features/tier2_retention.go +++ b/operator/test/klio/features/tier2_retention.go @@ -69,18 +69,19 @@ const ( // Tier2RetentionFeature defines a feature for testing tier2 backup and WAL retention. type Tier2RetentionFeature struct { - name string - setup types.StepFunc - teardown types.StepFunc - backups []*cnpgv1.Backup - klioServer *kliov1alpha1.Server - namespace string - keepLatest int - backupTimeout time.Duration - replicationTimeout time.Duration - checkInterval time.Duration - clusterName string - s3Prefix string + name string + setup types.StepFunc + teardown types.StepFunc + backups []*cnpgv1.Backup + klioServer *kliov1alpha1.Server + namespace string + keepLatest int + backupTimeout time.Duration + replicationTimeout time.Duration + checkInterval time.Duration + clusterName string + s3Prefix string + pluginConfigurationName string } // Tier2RetentionFeatureConfig holds the configuration for creating a tier2 retention feature test. @@ -109,6 +110,9 @@ type Tier2RetentionFeatureConfig struct { ClusterName string // S3Prefix is the S3 prefix used for tier2 storage. S3Prefix string + // PluginConfigurationName is the name of the PluginConfiguration, used to + // tighten the retention policy for the on-demand `klio retention apply` step. + PluginConfigurationName string } // NewTier2RetentionFeature creates a new Tier2RetentionFeature with the given configuration. @@ -124,18 +128,19 @@ func NewTier2RetentionFeature(config Tier2RetentionFeatureConfig) *Tier2Retentio } return &Tier2RetentionFeature{ - name: config.Name, - setup: config.Setup, - teardown: config.Teardown, - backups: config.Backups, - klioServer: config.KlioServer, - namespace: config.Namespace, - keepLatest: config.KeepLatest, - backupTimeout: config.BackupTimeout, - replicationTimeout: config.ReplicationTimeout, - checkInterval: config.CheckInterval, - clusterName: config.ClusterName, - s3Prefix: config.S3Prefix, + name: config.Name, + setup: config.Setup, + teardown: config.Teardown, + backups: config.Backups, + klioServer: config.KlioServer, + namespace: config.Namespace, + keepLatest: config.KeepLatest, + backupTimeout: config.BackupTimeout, + replicationTimeout: config.ReplicationTimeout, + checkInterval: config.CheckInterval, + clusterName: config.ClusterName, + s3Prefix: config.S3Prefix, + pluginConfigurationName: config.PluginConfigurationName, } } @@ -159,7 +164,8 @@ func (f *Tier2RetentionFeature) Setup() types.StepFunc { // was the backup the retention manager deleted (the newest survive). This // exercises the full path: PluginConfiguration CR -> operator -> klio-plugin // config -> CloseBackup GRPC -> NATS queue -> backup consumer -> Klio-managed -// retention. +// retention. It then tightens the policy to a single backup and applies it on +// demand with `klio retention apply`, verifying only the newest remains. // // 2. WAL Retention Verification: Monitors WAL directory count before and after // retention to verify WAL cleanup is occurring. This is a soft check (logs @@ -252,6 +258,50 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { t.Logf("[Level 1] PASSED: tier2 has exactly %d backup(s) and the oldest (%s) was deleted", f.keepLatest, oldestTier2BackupName) + // ========================================== + // Level 1b: On-demand retention (klio retention apply) + // ========================================== + // Tighten the tier2 retention to keep a single backup and apply it + // immediately with `klio retention apply`, without taking a new backup. + // Only the newest backup must remain afterwards. + newestBackupName := survivingNames[0] + t.Log("[Level 1b] On-demand retention: tightening tier2 retention to 1 and running `klio retention apply`") + updateTier2RetentionLatest(ctx, t, r, f.namespace, f.pluginConfigurationName, 1) + + instancePodName := f.backups[len(f.backups)-1].Status.InstanceID.PodName + require.NotEmpty(t, instancePodName, "Level 1b failed: backup instance pod name should be set") + + // The `klio retention apply` command sends the policy from the pod's + // mounted config, which the operator updates asynchronously after the + // PluginConfiguration change. Re-running the (idempotent) command until + // exactly one backup remains absorbs both the config propagation and the + // asynchronous consumer processing. + err = wait.For( + func(ctx context.Context) (bool, error) { + if applyErr := runRetentionApply(ctx, r, f.namespace, instancePodName); applyErr != nil { + t.Logf("[Level 1b] klio retention apply not ready yet: %v", applyErr) + + return false, nil + } + + names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + if listErr != nil { + return false, nil //nolint:nilerr + } + + return len(names) == 1, nil + }, + wait.WithTimeout(f.replicationTimeout), + wait.WithInterval(f.checkInterval), + ) + require.NoError(t, err, "Level 1b failed: on-demand retention did not converge to a single backup") + + finalNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + require.NoError(t, err, "Level 1b failed: could not list surviving tier2 backups") + require.Equal(t, []string{newestBackupName}, finalNames, + "Level 1b failed: only the newest backup should remain after `klio retention apply`") + t.Logf("[Level 1b] PASSED: only the newest backup (%s) remains after apply", newestBackupName) + // ========================================== // Level 2: WAL Retention Verification (Soft Check) // ========================================== @@ -406,6 +456,52 @@ func checkTier2HasBackups( } } +// updateTier2RetentionLatest fetches the PluginConfiguration and sets its +// tier2 retention policy to keep the given number of most recent backups. +func updateTier2RetentionLatest( + ctx context.Context, + t *testing.T, + r *resources.Resources, + namespace string, + pluginConfigurationName string, + latest int, +) { + t.Helper() + + var pc kliov1alpha1.PluginConfiguration + require.NoError(t, r.Get(ctx, pluginConfigurationName, namespace, &pc), + "failed to get PluginConfiguration %q", pluginConfigurationName) + + require.NotNil(t, pc.Spec.Tier2, "PluginConfiguration should have a tier2 section") + if pc.Spec.Tier2.RetentionPolicy == nil { + pc.Spec.Tier2.RetentionPolicy = &kliov1alpha1.RetentionPolicy{} + } + pc.Spec.Tier2.RetentionPolicy.Latest = latest + + require.NoError(t, r.Update(ctx, &pc), "failed to update PluginConfiguration retention") +} + +// runRetentionApply runs `klio retention apply` inside the klio-plugin sidecar +// of the given pod, using the pod's own archive config, to apply the configured +// retention policy on demand. +func runRetentionApply( + ctx context.Context, + r *resources.Resources, + namespace string, + podName string, +) error { + var stdout, stderr bytes.Buffer + applyCmd := []string{"klio", "retention", "apply", "--config", archiveConfigPath} + if err := r.ExecInPod( + ctx, namespace, podName, cnpgi.KlioPluginContainerName, applyCmd, &stdout, &stderr, + ); err != nil { + return fmt.Errorf("klio retention apply failed: %w; stdout: %s, stderr: %s", + err, stdout.String(), stderr.String()) + } + + return nil +} + // listTier2BackupNames returns the names of the backups currently present in // tier2 (those carrying the tier2 annotation), ordered newest first by their // start time. From 02b7cd25836e3ba1a073e111f678e5e3a6596ad8 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 03/23] fix(config): decode the tier1 retention policy from the client config The operator writes the client config with the `tier1_retention` key while the mapstructure tag said `retention`, so viper left the field nil and the tier1 retention policy never reached the server: tier1 retention was a no-op. Both synced copies of the struct are fixed and a decode test covers the retention blocks. Signed-off-by: Armando Ruocco --- core/pkg/config/client.go | 2 +- core/pkg/config/decode_test.go | 20 ++++++++++++++++++-- operator/pkg/config/client.go | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/core/pkg/config/client.go b/core/pkg/config/client.go index cbca13c0..7ed0fd53 100644 --- a/core/pkg/config/client.go +++ b/core/pkg/config/client.go @@ -33,7 +33,7 @@ type Data struct { Client ClientConfig `json:"client" mapstructure:"client"` // Tier1RetentionPolicy is the retention policy to be applied to tier1. - Tier1RetentionPolicy *RetentionPolicy `json:"tier1_retention,omitempty" mapstructure:"retention"` + Tier1RetentionPolicy *RetentionPolicy `json:"tier1_retention,omitempty" mapstructure:"tier1_retention"` // Tier2RetentionPolicy is the retention policy to be applied to tier2. Tier2RetentionPolicy *RetentionPolicy `json:"tier2_retention,omitempty" mapstructure:"tier2_retention"` diff --git a/core/pkg/config/decode_test.go b/core/pkg/config/decode_test.go index d1339b12..b5714ebf 100644 --- a/core/pkg/config/decode_test.go +++ b/core/pkg/config/decode_test.go @@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0 package config import ( + "reflect" "strings" "testing" @@ -93,6 +94,21 @@ source: }, }, }, + { + // The operator writes the retention blocks with the JSON tag keys, + // so the mapstructure tags must use the same names. + name: "retention policies", + yaml: ` +tier1_retention: + latest: 3 +tier2_retention: + latest: 5 +`, + want: Data{ + Tier1RetentionPolicy: &RetentionPolicy{Latest: 3}, + Tier2RetentionPolicy: &RetentionPolicy{Latest: 5}, + }, + }, { name: "empty YAML produces zero-value Data", yaml: `{}`, @@ -122,11 +138,11 @@ source: if got.Source != tt.want.Source { t.Errorf("Source = %+v, want %+v", got.Source, tt.want.Source) } - if got.Tier1RetentionPolicy != tt.want.Tier1RetentionPolicy { + if !reflect.DeepEqual(got.Tier1RetentionPolicy, tt.want.Tier1RetentionPolicy) { t.Errorf("Tier1RetentionPolicy = %v, want %v", got.Tier1RetentionPolicy, tt.want.Tier1RetentionPolicy) } - if got.Tier2RetentionPolicy != tt.want.Tier2RetentionPolicy { + if !reflect.DeepEqual(got.Tier2RetentionPolicy, tt.want.Tier2RetentionPolicy) { t.Errorf("Tier2RetentionPolicy = %v, want %v", got.Tier2RetentionPolicy, tt.want.Tier2RetentionPolicy) } diff --git a/operator/pkg/config/client.go b/operator/pkg/config/client.go index cbca13c0..7ed0fd53 100644 --- a/operator/pkg/config/client.go +++ b/operator/pkg/config/client.go @@ -33,7 +33,7 @@ type Data struct { Client ClientConfig `json:"client" mapstructure:"client"` // Tier1RetentionPolicy is the retention policy to be applied to tier1. - Tier1RetentionPolicy *RetentionPolicy `json:"tier1_retention,omitempty" mapstructure:"retention"` + Tier1RetentionPolicy *RetentionPolicy `json:"tier1_retention,omitempty" mapstructure:"tier1_retention"` // Tier2RetentionPolicy is the retention policy to be applied to tier2. Tier2RetentionPolicy *RetentionPolicy `json:"tier2_retention,omitempty" mapstructure:"tier2_retention"` From 857f5ce4598806aa5c62f7039c36d0937c81f1e2 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 04/23] fix(server): disable Kopia's own snapshot retention The `kopia repository create` command stores Kopia's default retention counters in the global policy and every `kopia snapshot create` applies the effective policy, so with the pins gone Kopia kept expiring tier1 snapshots behind Klio's back, per source, capped by its own buckets. The server now disables Kopia retention at start, before the Kopia servers come up: the global policy of each tier is set to keep everything, and the per-source policies left by earlier Klio versions are reset to inherit it. Signed-off-by: Armando Ruocco --- AGENTS.md | 5 +- core/cmd/server/server.go | 35 +++-- core/internal/kopia/retention.go | 151 +++++++++++++++++++ core/internal/kopia/retention_test.go | 69 +++++++++ documentation/web/docs/user/upgrade_notes.md | 4 + 5 files changed, 247 insertions(+), 17 deletions(-) create mode 100644 core/internal/kopia/retention.go create mode 100644 core/internal/kopia/retention_test.go diff --git a/AGENTS.md b/AGENTS.md index 27fad737..7bd23bba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -199,9 +199,10 @@ confirm after that warning. those steps: tier1/tier2 retention apply, tier2 relay/migrate, tier2 policy set, and tier1 unpin. This is a deliberate, contained exception — not a pattern to copy, and one that should be removed in the future. -- A second, narrower exception: `applyGlobalCompressionPolicy` in +- A second, narrower exception: `applyGlobalKopiaPolicies` in `core/cmd/server/server.go` sets the repository-wide (global) compression - policy with a raw `kopia.Client{ConfigFile: ...}`, before the tier's Kopia + policy and disables Kopia's own snapshot retention (Klio applies retention + itself) with a raw `kopia.Client{ConfigFile: ...}`, before the tier's Kopia server starts. This is safe only because no server is running yet to hold a stale cache. Do not reuse this pattern once the server is up. - A direct write that **rewrites the manifest of a live backup** MUST be followed diff --git a/core/cmd/server/server.go b/core/cmd/server/server.go index 76e2cdc0..958cf74e 100644 --- a/core/cmd/server/server.go +++ b/core/cmd/server/server.go @@ -34,14 +34,15 @@ import ( "github.com/cloudnative-pg/klio/core/pkg/config" ) -// applyGlobalCompressionPolicy sets the repository-wide (global) Kopia -// compression policy using the passed persistent config file. It is always -// applied, even when unconfigured, so that removing the compression section -// resets the global policy back to Kopia's built-in default instead of -// leaving a stale, previously-set policy in place. This runs before the -// Kopia servers start, so the direct write to the repository predates any -// server cache. -func applyGlobalCompressionPolicy( +// applyGlobalKopiaPolicies sets the repository-wide (global) Kopia +// compression policy using the passed persistent config file and disables +// Kopia's own snapshot retention, which Klio applies itself. The compression +// policy is always applied, even when unconfigured, so that removing the +// compression section resets the global policy back to Kopia's built-in +// default instead of leaving a stale, previously-set policy in place. This +// runs before the Kopia servers start, so the direct writes to the repository +// predate any server cache. +func applyGlobalKopiaPolicies( ctx context.Context, configFile string, compression config.CompressionPolicy, @@ -56,22 +57,26 @@ func applyGlobalCompressionPolicy( ConfigFile: configFile, } - return client.SetKopiaGlobalCompressionPolicy(ctx, kopia.CompressionPolicy{ + if err := client.SetKopiaGlobalCompressionPolicy(ctx, kopia.CompressionPolicy{ Algorithm: compression.Algorithm, MinSize: compression.MinSize, MaxSize: compression.MaxSize, - }) + }); err != nil { + return err + } + + return client.DisableKopiaRetention(ctx) } // setupTier1KopiaConfig connects the tier1 config file to the repository and -// applies the tier1 repository-wide compression policy. +// applies the tier1 repository-wide Kopia policies. func setupTier1KopiaConfig(ctx context.Context, configFile string, cfg *config.Tier1Config) error { if err := kopiaconfig.CreateTier1KopiaConfigFile(ctx, configFile, cfg); err != nil { return fmt.Errorf("error creating tier1 kopia config file: %w", err) } - if err := applyGlobalCompressionPolicy(ctx, configFile, cfg.Compression); err != nil { - return fmt.Errorf("error setting tier1 global compression policy: %w", err) + if err := applyGlobalKopiaPolicies(ctx, configFile, cfg.Compression); err != nil { + return fmt.Errorf("error setting tier1 global Kopia policies: %w", err) } return nil @@ -269,10 +274,10 @@ func runServer(ctx context.Context, opts serverOpts) error { // it on every restart, clobbering whatever the tier1-enabled server // that actually owns backups has configured. if opts.tier1 { - if err := applyGlobalCompressionPolicy( + if err := applyGlobalKopiaPolicies( ctx, tier2RWConfigFileName, opts.cfg.Tier2.Compression, ); err != nil { - return fmt.Errorf("error setting tier2 global compression policy: %w", err) + return fmt.Errorf("error setting tier2 global Kopia policies: %w", err) } } diff --git a/core/internal/kopia/retention.go b/core/internal/kopia/retention.go new file mode 100644 index 00000000..31c3db92 --- /dev/null +++ b/core/internal/kopia/retention.go @@ -0,0 +1,151 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package kopia + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + + "github.com/cloudnative-pg/machinery/pkg/log" +) + +// policyTarget is the source a Kopia policy applies to, as printed by +// `kopia policy list --json`. The global policy has every field empty. +type policyTarget struct { + Host string `json:"host"` + UserName string `json:"userName"` + Path string `json:"path"` +} + +// String formats the target the way `kopia policy set` parses it. +func (t policyTarget) String() string { + if t.Path == "" { + return t.UserName + "@" + t.Host + } + + return t.UserName + "@" + t.Host + ":" + t.Path +} + +func (t policyTarget) isGlobal() bool { + return t.Host == "" && t.UserName == "" && t.Path == "" +} + +// DisableKopiaRetention turns off Kopia's own snapshot retention in the +// repository, since Klio applies retention itself. `kopia repository create` +// stores Kopia's default retention counters in the global policy and every +// `kopia snapshot create` expires snapshots against the effective policy, +// so without this Kopia keeps deleting backups behind Klio's back. +// +// An all-zero retention policy is Kopia's encoding for "keep everything"; +// the per-source policies left by earlier Klio versions are reset to inherit +// the global one. It must run before the Kopia server starts, so the direct +// writes predate any server cache. +func (s *Client) DisableKopiaRetention(ctx context.Context) error { + if err := s.setPolicyRetention(ctx, "--global", "0"); err != nil { + return fmt.Errorf("while disabling global Kopia retention: %w", err) + } + + targets, err := s.listPolicyTargets(ctx) + if err != nil { + return err + } + + for _, target := range targets { + if target.isGlobal() { + continue + } + + if err := s.setPolicyRetention(ctx, target.String(), "inherit"); err != nil { + return fmt.Errorf("while resetting Kopia retention of %q: %w", target.String(), err) + } + } + + return nil +} + +func (s *Client) setPolicyRetention(ctx context.Context, policyTarget string, value string) error { + args := buildRetentionPolicyArgs(s.ConfigFile, policyTarget, value) + + log.FromContext(ctx).Info("Setting Kopia retention policy", "args", args) + + setPolicyCmd := exec.CommandContext(ctx, s.KopiaBinary, args...) //nolint:gosec + setPolicyCmd.Env = s.kopiaEnvironmentVariables() + + return RunWithLogCapture(ctx, setPolicyCmd, nil) +} + +func buildRetentionPolicyArgs(configFile, policyTarget, value string) []string { + args := []string{ + "policy", + "set", + "--config-file=" + configFile, + "--disable-file-logging", + } + + for _, counter := range kopiaRetentionCounters() { + args = append(args, counter+"="+value) + } + + return append(args, policyTarget) +} + +// kopiaRetentionCounters are the retention flags of `kopia policy set`. +func kopiaRetentionCounters() []string { + return []string{ + "--keep-latest", + "--keep-hourly", + "--keep-daily", + "--keep-weekly", + "--keep-monthly", + "--keep-annual", + } +} + +func (s *Client) listPolicyTargets(ctx context.Context) ([]policyTarget, error) { + listCmd := exec.CommandContext(ctx, s.KopiaBinary, //nolint:gosec + "policy", "list", "--json", "--config-file="+s.ConfigFile, "--disable-file-logging") + listCmd.Env = s.kopiaEnvironmentVariables() + + var stdout bytes.Buffer + if err := RunWithLogCapture(ctx, listCmd, &stdout); err != nil { + return nil, fmt.Errorf("while listing Kopia policies: %w", err) + } + + return parsePolicyTargets(stdout.Bytes()) +} + +func parsePolicyTargets(raw []byte) ([]policyTarget, error) { + var policies []struct { + Target policyTarget `json:"target"` + } + if err := json.Unmarshal(raw, &policies); err != nil { + return nil, fmt.Errorf("while unmarshalling Kopia policy list %q: %w", string(raw), err) + } + + targets := make([]policyTarget, len(policies)) + for i := range policies { + targets[i] = policies[i].Target + } + + return targets, nil +} diff --git a/core/internal/kopia/retention_test.go b/core/internal/kopia/retention_test.go new file mode 100644 index 00000000..c2f2301d --- /dev/null +++ b/core/internal/kopia/retention_test.go @@ -0,0 +1,69 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package kopia + +import ( + "slices" + "testing" +) + +func TestBuildRetentionPolicyArgs(t *testing.T) { + args := buildRetentionPolicyArgs("/etc/kopia/config", "--global", "0") + + for _, counter := range kopiaRetentionCounters() { + assertArgContains(t, args, counter+"=0") + } + assertArgContains(t, args, "--config-file=/etc/kopia/config") + if args[len(args)-1] != "--global" { + t.Errorf("expected the target as last arg, got %v", args) + } +} + +func TestParsePolicyTargets(t *testing.T) { + // Trimmed `kopia policy list --json` output: the global policy plus a + // per-source one, with the unrelated policy fields left in place. + raw := []byte(`[ + {"id":"global","target":{"host":"","userName":"","path":""},"retention":{"keepLatest":10}}, + {"id":"abc","target":{"host":"cluster","userName":"user","path":"/pgdata"},"compression":{}}, + {"id":"def","target":{"host":"cluster","userName":"user","path":""}} + ]`) + + targets, err := parsePolicyTargets(raw) + if err != nil { + t.Fatalf("parsePolicyTargets() error = %v", err) + } + + var got []string + for _, target := range targets { + if target.isGlobal() { + continue + } + got = append(got, target.String()) + } + + want := []string{"user@cluster:/pgdata", "user@cluster"} + if !slices.Equal(got, want) { + t.Errorf("targets = %v, want %v", got, want) + } + + if _, err := parsePolicyTargets([]byte("not json")); err == nil { + t.Error("expected an error on malformed output") + } +} diff --git a/documentation/web/docs/user/upgrade_notes.md b/documentation/web/docs/user/upgrade_notes.md index 41b0b78a..6fcb766a 100644 --- a/documentation/web/docs/user/upgrade_notes.md +++ b/documentation/web/docs/user/upgrade_notes.md @@ -21,6 +21,10 @@ instead of being delegated to Kopia. field, which keeps the given number of most recent backups. Update any `tier1.retention` and `tier2.retention` blocks accordingly. Omit the block to keep every backup; when present, `latest` must be at least `1`. +- Kopia's own retention is disabled: on start, the Klio server sets the + global Kopia retention policy of each tier to keep every snapshot and + resets the per-source retention policies written by earlier versions to + inherit it. No manual action is required. - The `klio retention` command has been removed. Retention is configured only through the `PluginConfiguration`. From 74d24303883c4d4ef2adf2422878d19b2584befb Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 05/23] test(retention): cover applyRetention with a fake client The deletion glue (catalog listing, policy evaluation, keep guard, error joining) had no unit test. Narrow the client parameter to the two methods it uses so a fake can drive it. Signed-off-by: Armando Ruocco --- core/internal/consumer/backup.go | 8 +- core/internal/consumer/retention_test.go | 115 +++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 core/internal/consumer/retention_test.go diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index 7383bd46..1eed3c2b 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -335,6 +335,12 @@ func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask) erro return nil } +// retentionClient is the subset of the Kopia client that applyRetention needs. +type retentionClient interface { + ListBackups(ctx context.Context, hostname string) (klioclient.BackupList, error) + DeleteBackup(ctx context.Context, hostname string, name string) error +} + // applyRetention deletes the base backups of a cluster that fall outside the // given policy, evaluated against Klio's own catalog. The optional keep // predicate protects a backup from deletion even when the policy expired it @@ -342,7 +348,7 @@ func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask) erro // policy deletes nothing. func (d *Backup) applyRetention( ctx context.Context, - client *klioclientkopia.Connection, + client retentionClient, clusterName string, policy retention.Policy, keep func(name string) bool, diff --git a/core/internal/consumer/retention_test.go b/core/internal/consumer/retention_test.go new file mode 100644 index 00000000..22ff186c --- /dev/null +++ b/core/internal/consumer/retention_test.go @@ -0,0 +1,115 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package consumer + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/cloudnative-pg/klio/core/internal/client/klioclient" + "github.com/cloudnative-pg/klio/core/pkg/retention" +) + +var errFakeDelete = errors.New("fake delete failure") + +type fakeRetentionClient struct { + backups klioclient.BackupList + listErr error + failOn string + deleted []string +} + +func (f *fakeRetentionClient) ListBackups(_ context.Context, _ string) (klioclient.BackupList, error) { + return f.backups, f.listErr +} + +func (f *fakeRetentionClient) DeleteBackup(_ context.Context, _ string, name string) error { + if name == f.failOn { + return errFakeDelete + } + f.deleted = append(f.deleted, name) + + return nil +} + +func TestApplyRetention(t *testing.T) { + catalog := klioclient.BackupList{ + {Name: "b1", StartedAt: 100}, + {Name: "b2", StartedAt: 200}, + {Name: "b3", StartedAt: 300}, + } + + tests := []struct { + name string + client *fakeRetentionClient + policy retention.Policy + keep func(string) bool + wantDeleted []string + wantErr error + }{ + { + name: "zero policy deletes nothing", + client: &fakeRetentionClient{backups: catalog}, + }, + { + name: "latest keeps newest and deletes the rest", + client: &fakeRetentionClient{backups: catalog}, + policy: retention.Policy{Latest: 1}, + wantDeleted: []string{"b1", "b2"}, + }, + { + name: "keep predicate protects an expired backup", + client: &fakeRetentionClient{backups: catalog}, + policy: retention.Policy{Latest: 1}, + keep: func(name string) bool { return name == "b1" }, + wantDeleted: []string{"b2"}, + }, + { + name: "one failing delete does not stop the others", + client: &fakeRetentionClient{backups: catalog, failOn: "b1"}, + policy: retention.Policy{Latest: 1}, + wantDeleted: []string{"b2"}, + wantErr: errFakeDelete, + }, + { + name: "list failure deletes nothing", + client: &fakeRetentionClient{listErr: errFakeDelete}, + policy: retention.Policy{Latest: 1}, + wantErr: errFakeDelete, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Backup{} + err := d.applyRetention(context.Background(), tt.client, "cluster", tt.policy, tt.keep) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("applyRetention() error = %v, want %v", err, tt.wantErr) + } + + slices.Sort(tt.client.deleted) + if !slices.Equal(tt.client.deleted, tt.wantDeleted) { + t.Errorf("deleted = %v, want %v", tt.client.deleted, tt.wantDeleted) + } + }) + } +} From 6dbfbf5045fecf7b809cd080d385fe35d4a6718d Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 06/23] fix(retention): reject an empty or invalid cluster name in ApplyRetention An empty cluster name is a wildcard for the backup listing, so a maintenance task carrying one evaluated the policy over every cluster's catalog and then failed on each delete, blocking the single-ack consumer through its retry budget. The RPC now rejects an empty or invalid name and the consumer refuses to apply retention without one. Signed-off-by: Armando Ruocco --- core/internal/consumer/backup.go | 10 ++++++++++ core/internal/server/walserver/backup.go | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index 1eed3c2b..e6a03512 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -43,6 +43,10 @@ import ( // dead-lettered) so the misconfiguration is surfaced. var errTier2NotConfigured = errors.New("backup requested tier2 relay but the server has no tier2 configured") +// errEmptyClusterName is returned when retention is requested without a +// cluster name, which would otherwise match every cluster's backups. +var errEmptyClusterName = errors.New("retention requires a cluster name") + // backupSteps is implemented by *Backup in production and by a test stub in // unit tests. It covers the five steps that processBackup orchestrates so // that the orchestration logic can be exercised without real Kopia clients. @@ -355,6 +359,12 @@ func (d *Backup) applyRetention( ) error { contextLogger := log.FromContext(ctx) + // ListBackups treats an empty host as a wildcard: the catalog would span + // every cluster and the policy would be applied across all of them. + if clusterName == "" { + return errEmptyClusterName + } + backups, err := client.ListBackups(ctx, clusterName) if err != nil { return fmt.Errorf("while listing backups for cluster %q: %w", clusterName, err) diff --git a/core/internal/server/walserver/backup.go b/core/internal/server/walserver/backup.go index e200b67b..617340d5 100644 --- a/core/internal/server/walserver/backup.go +++ b/core/internal/server/walserver/backup.go @@ -32,6 +32,7 @@ import ( "github.com/cloudnative-pg/klio/core/internal/grpc" "github.com/cloudnative-pg/klio/core/internal/kopia" "github.com/cloudnative-pg/klio/core/internal/queue" + "github.com/cloudnative-pg/klio/core/internal/repository" "github.com/cloudnative-pg/klio/core/pkg/retention" ) @@ -83,6 +84,15 @@ func (w *Implementation) ApplyRetention( return nil, status.Errorf(codes.Internal, "queue service is uninitialized") } + // An empty name would list every cluster's catalog, so reject it along + // with any invalid path component. + if request.GetClusterName() == "" { + return nil, status.Errorf(codes.InvalidArgument, "cluster name is required") + } + if err := repository.ValidatePathComponent(request.GetClusterName()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid cluster name: %v", err) + } + tier1Policy := parseRetentionPolicy(ctx, "tier1", request.GetTier1RetentionPolicy()) tier2Policy := parseRetentionPolicy(ctx, "tier2", request.GetTier2RetentionPolicy()) From 5f1458c10b1bbf9d5c0a23a66f2cab1e70d48e5f Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 07/23] fix(retention): retry a maintenance-only task when tier1 maintenance fails Best-effort tier1 maintenance is right after a backup, which must not fail on it. A maintenance-only task has no backup to protect: the retention run is the whole task, so a tier1 failure was logged once and acked as done while the client had been told it was scheduled. It now fails the task so the queue retries it. Signed-off-by: Armando Ruocco --- core/internal/consumer/backup.go | 12 +++++------- core/internal/consumer/processbackup_test.go | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index e6a03512..b5c59b31 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -245,8 +245,8 @@ func (d *Backup) relayAndMaintain(ctx context.Context, task *queue.BackupTask, e // maintainOnly applies the per-tier retention policies to a cluster without a // new backup, driven by a maintenance-only task. tier2 maintenance runs first // (when tier2 is configured) so the tier1 guard sees an up-to-date tier2 -// catalog before tier1 deletes anything; a tier2 failure is fatal so the task -// is retried, while tier1 maintenance is best-effort. +// catalog before tier1 deletes anything; a failure on either tier is fatal so +// the task is retried. func (d *Backup) maintainOnly(ctx context.Context, task *queue.BackupTask) error { contextLogger := log.FromContext(ctx) contextLogger.Info("Applying on-demand retention", "cluster", task.ClusterName) @@ -257,11 +257,9 @@ func (d *Backup) maintainOnly(ctx context.Context, task *queue.BackupTask) error } } - if err := d.steps.maintainTier1(ctx, task); err != nil { - contextLogger.Error(err, "Error while applying tier1 maintenance, skipping") - } - - return nil + // Unlike the post-backup path there is no backup to protect here: the + // retention run is the whole task, so a failure must be retried. + return d.steps.maintainTier1(ctx, task) } // relayTier2 migrates the cluster's backups to tier2 and verifies them there. diff --git a/core/internal/consumer/processbackup_test.go b/core/internal/consumer/processbackup_test.go index f307f530..8517c287 100644 --- a/core/internal/consumer/processbackup_test.go +++ b/core/internal/consumer/processbackup_test.go @@ -182,6 +182,13 @@ func TestProcessBackup(t *testing.T) { wantMaintain2: true, wantMaintain: true, }, + { + name: "maintenance-only tier1 failure is retried", + maintenanceOnly: true, + maintainErr: errBoom, + wantErr: true, + wantMaintain: true, + }, { name: "maintenance-only tier2 failure is retried before tier1", maintenanceOnly: true, From ef3fa6e6fbaad56709fa5ece3d6a875070c5573a Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 08/23] fix(queue): do not purge the backup dead-letter queue on maintenance tasks The purge after a successful task exists because a completed backup makes the earlier failures moot. A maintenance-only task reuses the same handler, so an on-demand retention run erased dead-lettered backup tasks without any backup having succeeded. Signed-off-by: Armando Ruocco --- core/internal/queue/backup.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/internal/queue/backup.go b/core/internal/queue/backup.go index b91fb0fa..b1a6bdda 100644 --- a/core/internal/queue/backup.go +++ b/core/internal/queue/backup.go @@ -90,6 +90,12 @@ func (q *Conn) ConsumeBackupReceivedMessages(ctx context.Context, handler Backup return err } + // Only a successful backup makes the earlier failures moot; an + // on-demand retention run must not erase them. + if t.MaintenanceOnly { + return nil + } + if err := q.purgeBackupDLQEntries(ctx, t.ClusterName); err != nil { log.FromContext(ctx).Error( err, From d12368f9b61ca9d42e19fc3095551fc927bf5fcc Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 09/23] fix(backup): delete the metadata snapshot of a backup last The snapshots of a backup were deleted in listing order. A deletion interrupted after the metadata snapshot left the data snapshots orphaned and invisible to the catalog, so retention could never finish the job. Deleting the metadata last keeps a partially deleted backup listed until the next run completes it. Signed-off-by: Armando Ruocco --- core/internal/client/klioclient/kopia/delete.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/internal/client/klioclient/kopia/delete.go b/core/internal/client/klioclient/kopia/delete.go index 775f3bbf..3e524313 100644 --- a/core/internal/client/klioclient/kopia/delete.go +++ b/core/internal/client/klioclient/kopia/delete.go @@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0 package kopia import ( + "cmp" "context" "errors" "fmt" @@ -96,6 +97,14 @@ func deleteBackupSnapshots(ctx context.Context, store snapshotStore, hostname, n return retryErr } +func isMetadataSnapshot(m kopia.Manifest) int { + if m.Tags[klioclient.BackupContentTagName] == "metadata" { + return 1 + } + + return 0 +} + // deleteSnapshots deletes every given entry, joining and returning any // deletion errors. func deleteSnapshots( @@ -105,6 +114,13 @@ func deleteSnapshots( ) error { contextLogger := log.FromContext(ctx) + // The metadata snapshot is what makes a backup visible in the catalog: + // deleting it last keeps a partially deleted backup listed, so the next + // retention run can finish the job instead of leaving orphaned data. + slices.SortStableFunc(entries, func(a, b kopia.Manifest) int { + return cmp.Compare(isMetadataSnapshot(a), isMetadataSnapshot(b)) + }) + var err error for _, entry := range entries { From 90b0712ba2e742e85a6a0a9976342e0116554e37 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 10/23] fix(operator): treat a pruned retention block as no policy A PluginConfiguration stored before the `latest` field existed reads back with `latest: 0` once the old keys are pruned. The operator emitted a retention block for it, the client rejected the config and every backup of the cluster failed until the resource was edited. Such a block now converts to "no policy", keeping every backup; the CRD validation still rejects `latest` below 1 on write. Signed-off-by: Armando Ruocco --- operator/internal/klioconfig/config.go | 5 ++++- operator/internal/klioconfig/config_test.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/operator/internal/klioconfig/config.go b/operator/internal/klioconfig/config.go index 79dc36d8..e0dad1e1 100644 --- a/operator/internal/klioconfig/config.go +++ b/operator/internal/klioconfig/config.go @@ -157,7 +157,10 @@ func GenerateConfig( } func convertRetentionPolicy(p *kliov1alpha1.RetentionPolicy) *config.RetentionPolicy { - if p == nil { + // A stored object written before the "latest" field existed reads back + // with Latest 0 (the old keys are pruned): treat it as no policy instead + // of emitting a config that fails client validation. + if p == nil || p.Latest < 1 { return nil } diff --git a/operator/internal/klioconfig/config_test.go b/operator/internal/klioconfig/config_test.go index 286c9558..f4e7ffc5 100644 --- a/operator/internal/klioconfig/config_test.go +++ b/operator/internal/klioconfig/config_test.go @@ -341,6 +341,10 @@ func TestConvertRetentionPolicy(t *testing.T) { assert.Nil(t, result) }) + t.Run("latest below 1 returns nil", func(t *testing.T) { + assert.Nil(t, convertRetentionPolicy(&kliov1alpha1.RetentionPolicy{})) + }) + t.Run("latest is set", func(t *testing.T) { input := &kliov1alpha1.RetentionPolicy{ Latest: 5, From 57464a8d8f7d54619878fe4e107debfe56e9b04c Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 11/23] test(e2e): list only the cluster's backups in the tier2 retention feature The helper read every backup known to the Klio server, so the exact-count assertions would fail as soon as another cluster shared the server. Signed-off-by: Armando Ruocco --- operator/test/klio/features/tier2_retention.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/operator/test/klio/features/tier2_retention.go b/operator/test/klio/features/tier2_retention.go index e85609db..a82d9b98 100644 --- a/operator/test/klio/features/tier2_retention.go +++ b/operator/test/klio/features/tier2_retention.go @@ -222,7 +222,7 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { // After the first backup, record the WAL directory count as baseline // and remember which backup reached tier2 first (the oldest one). if i == 0 { - names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) require.NoError(t, listErr, "failed to list tier2 backups after the first backup") require.Len(t, names, 1, "expected exactly one tier2 backup after the first backup") oldestTier2BackupName = names[0] @@ -251,7 +251,7 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { ) require.NoError(t, err, "Level 1 failed: tier2 backup count verification failed") - survivingNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + survivingNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) require.NoError(t, err, "Level 1 failed: could not list surviving tier2 backups") require.NotContains(t, survivingNames, oldestTier2BackupName, "Level 1 failed: the oldest backup should have been deleted by the retention manager") @@ -284,7 +284,7 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { return false, nil } - names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) if listErr != nil { return false, nil //nolint:nilerr } @@ -296,7 +296,7 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { ) require.NoError(t, err, "Level 1b failed: on-demand retention did not converge to a single backup") - finalNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name) + finalNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) require.NoError(t, err, "Level 1b failed: could not list surviving tier2 backups") require.Equal(t, []string{newestBackupName}, finalNames, "Level 1b failed: only the newest backup should remain after `klio retention apply`") @@ -510,6 +510,7 @@ func listTier2BackupNames( r *resources.Resources, namespace string, serverName string, + clusterName string, ) ([]string, error) { podName := serverName + klioPodSuffix @@ -521,6 +522,7 @@ func listTier2BackupNames( type backupMetadata struct { Name string `json:"name"` + ClusterName string `json:"clusterName"` StartedAt int64 `json:"startedAt"` Annotations map[string]string `json:"annotations,omitempty"` } @@ -536,6 +538,9 @@ func listTier2BackupNames( names := make([]string, 0, len(backups)) for i := range backups { + if backups[i].ClusterName != clusterName { + continue + } if backups[i].Annotations[tier2AnnotationName] == presentAnnotationValue { names = append(names, backups[i].Name) } From 78ee194d60570a2fa8959e4cea2dbbd27e29b452 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:13:48 +0200 Subject: [PATCH 12/23] chore(retention): drop leftovers of the Kopia pin based flow Remove the consumer options only read by the deleted tier1 server refresh, fix a comment naming a method that no longer exists, correct the upgrade note (only `klio retention get` and `set` went away, `apply` is new) and update AGENTS.md, which still described the pins and the tier1 unpin. Signed-off-by: Armando Ruocco --- AGENTS.md | 43 ++++++++++---------- core/internal/consumer/backup.go | 6 --- core/internal/server/backupconsumer.go | 18 ++++---- core/pkg/retention/retention.go | 4 +- documentation/web/docs/user/upgrade_notes.md | 5 ++- 5 files changed, 34 insertions(+), 42 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7bd23bba..6aaa0494 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,15 +190,16 @@ first.** Explain that it bypasses the Kopia server cache, name the race it can introduce, and propose the server-routed alternative. Only proceed if they confirm after that warning. -- Client- and sidecar-driven paths (backup upload, delete, retention set, +- Client- and sidecar-driven paths (backup upload, delete, retention apply, restore, list; everything under `core/cmd/*`) already route through the server via `MultiConnect`/`ConnectTier1`/`ConnectTier2`. Keep them that way — never convert one of these to a direct write. - The **only** component that writes directly is the server-side backup consumer (`core/internal/consumer/`), and only because it has no server connection for - those steps: tier1/tier2 retention apply, tier2 relay/migrate, tier2 policy - set, and tier1 unpin. This is a deliberate, contained exception — not a pattern - to copy, and one that should be removed in the future. + those steps: tier1/tier2 retention apply (snapshot deletes), tier2 + relay/migrate, and tier2 compression policy set. This is a deliberate, + contained exception — not a pattern to copy, and one that should be removed + in the future. - A second, narrower exception: `applyGlobalKopiaPolicies` in `core/cmd/server/server.go` sets the repository-wide (global) compression policy and disables Kopia's own snapshot retention (Klio applies retention @@ -206,13 +207,14 @@ confirm after that warning. server starts. This is safe only because no server is running yet to hold a stale cache. Do not reuse this pattern once the server is up. - A direct write that **rewrites the manifest of a live backup** MUST be followed - by `refreshTier1KopiaServer` / `refreshTier2KopiaServer` so the servers - reconcile their caches; skipping the refresh is a bug. The tier1 unpin is the - canonical case: `kopia snapshot pin` rewrites the snapshot manifest to a *new* - ID and deletes the old one, so without a refresh the server keeps serving the - now-deleted ID for a backup that still exists, and a later client - `klio backup delete` asks Kopia to delete an ID that no longer matches - anything: the command fails and the real backup (and its WALs) stay pinned. + by `refreshTier2KopiaServer` (or an equivalent tier1 refresh, should such a + write come back) so the servers reconcile their caches; skipping the refresh + is a bug. The former tier1 unpin was the canonical case: `kopia snapshot pin` + rewrites the snapshot manifest to a *new* ID and deletes the old one, so + without a refresh the server kept serving the now-deleted ID for a backup + that still existed, and a later client `klio backup delete` asked Kopia to + delete an ID that no longer matched anything: the command failed and the + real backup (and its WALs) stayed pinned. Klio no longer pins snapshots. - A direct write that only **deletes** snapshots (the tier1/tier2 retention apply) does **not** need a refresh: it removes IDs the server may still list, but it never rewrites a live backup's ID, and WAL retention is recomputed from @@ -227,10 +229,10 @@ refresh of the affected tier. ### Snapshot identity: manifest ID vs root object ID -A snapshot's **manifest ID is not a stable identity**. `kopia snapshot pin` -(the tier1 unpin above) rewrites a snapshot's manifest under a new ID and -deletes the old one, so any code that lists snapshots and then acts on them a -moment later can be holding an ID that no longer exists. Pick the identity by +A snapshot's **manifest ID is not a stable identity**. Some Kopia writes +(`kopia snapshot pin`, for one) rewrite a snapshot's manifest under a new ID +and delete the old one, so any code that lists snapshots and then acts on them +a moment later can be holding an ID that no longer exists. Pick the identity by what the operation does: - **Reads that must survive a concurrent rewrite** use the root object ID @@ -245,13 +247,10 @@ what the operation does: so deleting one backup by root ID can take another backup's snapshot with it. Delete by manifest ID, and on failure re-list and retry so a concurrent rewrite is picked up (`DeleteBackup` in the same package). -- **The tier1 unpin is a write, not a read, and knowingly accepts the same - collision as delete.** The consumer's `getPinnedSnapshots`/`maintainTier2` - (`core/internal/consumer/backup.go`) also targets the root object ID, so a - root shared with another backup gets unpinned too. This is tolerated only - because the step is best-effort and the affected snapshot would be unpinned - anyway on the next tier2 migration — it is not a safe pattern to copy for - anything that isn't equally tolerant of that collision. +- **Writes that target the root object ID hit the same collision as delete.** + The former tier1 unpin did this knowingly, tolerated only because the step + was best-effort. Any new write keyed by root object ID needs the same + analysis and must be equally tolerant of acting on another backup's snapshot. ### Dagger caching issues diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index b5c59b31..26c3e761 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -77,12 +77,6 @@ type BackupOptions struct { // A config file to connect to tier 1 Tier1KopiaConfig string - // Tier1ServerAddress is the address of the tier 1 Kopia server. - Tier1ServerAddress string - - // Tier1ServerCertificateFingerprint is the SHA256 fingerprint of the tier 1 server certificate. - Tier1ServerCertificateFingerprint string - // A config file to connect to tier 2 Tier2KopiaConfig string diff --git a/core/internal/server/backupconsumer.go b/core/internal/server/backupconsumer.go index bfeba3f5..f369a76b 100644 --- a/core/internal/server/backupconsumer.go +++ b/core/internal/server/backupconsumer.go @@ -78,8 +78,8 @@ func (s *BackupConsumer) Serve(ctx context.Context) error { return fmt.Errorf("failed to open tier1 WAL repository: %w", err) } - // Extract the certificate fingerprint for the Kopia servers. Tier1 and - // tier2 share the same server certificate. + // Extract the certificate fingerprint for the tier2 Kopia server, which + // shares the tier1 server certificate. certificateFingerprint, err := kopia.ExtractSHA256CertificateFingerprint( s.Config.TLS.TLSCert) if err != nil { @@ -87,14 +87,12 @@ func (s *BackupConsumer) Serve(ctx context.Context) error { } backupOptions := &consumer.BackupOptions{ - Queue: queueConnection, - Tier1KopiaConfig: s.Tier1KopiaConfigFile, - Tier1ServerAddress: "https://" + s.Config.Tier1.Base.ListenAddress, - Tier1ServerCertificateFingerprint: certificateFingerprint, - CacheDirectory: s.Config.Tier1.Base.CacheDirectory, - RunID: s.RunID, - RunSecret: s.RunSecret, - Tier1WALRepository: tier1WALRepository, + Queue: queueConnection, + Tier1KopiaConfig: s.Tier1KopiaConfigFile, + CacheDirectory: s.Config.Tier1.Base.CacheDirectory, + RunID: s.RunID, + RunSecret: s.RunSecret, + Tier1WALRepository: tier1WALRepository, } // When tier2 is configured, wire the tier2 connections so the consumer diff --git a/core/pkg/retention/retention.go b/core/pkg/retention/retention.go index f553de97..cfe83d29 100644 --- a/core/pkg/retention/retention.go +++ b/core/pkg/retention/retention.go @@ -54,8 +54,8 @@ type Backup struct { // returned more than once. // // For the "latest" criterion the N most recent backups (ordered by StartedAt, -// most recent first) are kept and every older backup is expired. A policy that -// keeps everything (see Policy.IsZero) yields no expired backups. +// most recent first) are kept and every older backup is expired. A policy with +// Latest below 1 keeps everything and yields no expired backups. func Evaluate(catalog []Backup, policy Policy) []Backup { // A count below 1 means no retention is configured (see Policy): keep // everything. This also guards the slice bound below. diff --git a/documentation/web/docs/user/upgrade_notes.md b/documentation/web/docs/user/upgrade_notes.md index 6fcb766a..e1b5f3c2 100644 --- a/documentation/web/docs/user/upgrade_notes.md +++ b/documentation/web/docs/user/upgrade_notes.md @@ -25,8 +25,9 @@ instead of being delegated to Kopia. global Kopia retention policy of each tier to keep every snapshot and resets the per-source retention policies written by earlier versions to inherit it. No manual action is required. -- The `klio retention` command has been removed. Retention is configured only - through the `PluginConfiguration`. +- The `klio retention get` and `klio retention set` commands have been + removed. Retention is configured only through the `PluginConfiguration`; + `klio retention apply` enforces it on demand. ## 0.0.20 to 0.0.21 From 945f086e89296552627a24e3215f6d28fe09c6d8 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:03:39 +0200 Subject: [PATCH 13/23] fix(retention): require a complete tier2 copy before tier1 deletes a backup The tier1 retention guard considered a backup safe to delete as soon as its metadata snapshot appeared on tier2. The relay is a single snapshot migration with no ordering between a backup's parts, so a relay that dies midway can leave the metadata on tier2 without the data, and the next maintenance run would then delete the only complete copy on tier1. The guard now lists the snapshots of both tiers and keeps a backup until every tier1 snapshot of it (pgdata, metadata, control data, tablespaces) has a counterpart on tier2. Signed-off-by: Armando Ruocco --- core/internal/consumer/maintenance.go | 57 ++++++++++++++++----- core/internal/consumer/maintenance_test.go | 58 ++++++++++++++++++---- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/core/internal/consumer/maintenance.go b/core/internal/consumer/maintenance.go index 36838bc4..92dcc52e 100644 --- a/core/internal/consumer/maintenance.go +++ b/core/internal/consumer/maintenance.go @@ -28,6 +28,7 @@ import ( "github.com/cloudnative-pg/machinery/pkg/stringset" "github.com/cloudnative-pg/klio/core/internal/client/klioclient" + "github.com/cloudnative-pg/klio/core/internal/kopia" "github.com/cloudnative-pg/klio/core/internal/opentelemetry" "github.com/cloudnative-pg/klio/core/internal/queue" "github.com/cloudnative-pg/klio/core/internal/repository" @@ -73,34 +74,66 @@ func (d *Backup) runTier1Retention(ctx context.Context, task *queue.BackupTask) } // tier1RetentionGuard returns a predicate that reports whether a tier1 backup -// must be kept because it has not yet been migrated to tier2. When tier2 is not -// configured there is nothing to protect and the predicate is nil. +// must be kept because it has not yet been fully migrated to tier2. When tier2 +// is not configured there is nothing to protect and the predicate is nil. func (d *Backup) tier1RetentionGuard(ctx context.Context, clusterName string) (func(name string) bool, error) { if !d.tier2Enabled { return nil, nil } - tier2Backups, err := d.tier2Client.ListBackups(ctx, clusterName) + tier1Snapshots, err := d.listManifests(ctx, clusterName) if err != nil { return nil, fmt.Errorf( - "while listing tier2 backups to guard tier1 retention for cluster %q: %w", clusterName, err) + "while listing tier1 snapshots to guard tier1 retention for cluster %q: %w", clusterName, err) } - return keepUntilOnTier2(tier2Backups), nil + tier2Snapshots, err := d.tier2Kopia.ListSnapshots(ctx, nil, log.FromContext(ctx).Info) + if err != nil { + return nil, fmt.Errorf( + "while listing tier2 snapshots to guard tier1 retention for cluster %q: %w", clusterName, err) + } + + return keepUntilOnTier2(tier1Snapshots, tier2Snapshots), nil } // keepUntilOnTier2 returns a predicate that reports whether a tier1 backup must -// be kept because it is not yet present on tier2. A backup becomes deletable -// once it appears in the tier2 catalog. -func keepUntilOnTier2(tier2Backups klioclient.BackupList) func(name string) bool { +// be kept because it is not yet complete on tier2. The relay is a single +// snapshot migration with no ordering between a backup's parts, so the tier2 +// metadata snapshot alone does not prove the data is there: a backup is +// deletable only when every tier1 snapshot of it has a counterpart on tier2. +func keepUntilOnTier2(tier1Snapshots, tier2Snapshots []kopia.Manifest) func(name string) bool { onTier2 := stringset.New() - for i := range tier2Backups { - onTier2.Put(tier2Backups[i].Name) + for i := range tier2Snapshots { + onTier2.Put(snapshotPartKey(tier2Snapshots[i])) } - return func(name string) bool { - return !onTier2.Has(name) + incomplete := stringset.New() + for i := range tier1Snapshots { + name := tier1Snapshots[i].Tags[klioclient.BackupNameTagName] + if name == "" { + continue + } + + if !onTier2.Has(snapshotPartKey(tier1Snapshots[i])) { + incomplete.Put(name) + } } + + return incomplete.Has +} + +// snapshotPartKey identifies one part of a backup (pgdata, metadata, control +// data or a tablespace) independently of the repository it lives in. The +// migration preserves both the source and the tags, so the key matches across +// tiers. +func snapshotPartKey(m kopia.Manifest) string { + return strings.Join([]string{ + m.Source.Host, + m.Source.Path, + m.Tags[klioclient.BackupNameTagName], + m.Tags[klioclient.BackupContentTagName], + m.Tags[klioclient.TablespaceNameTagName], + }, "\x00") } // applyTier1WALRetention drops the tier1 WAL files that are no longer required diff --git a/core/internal/consumer/maintenance_test.go b/core/internal/consumer/maintenance_test.go index 7af48367..49460292 100644 --- a/core/internal/consumer/maintenance_test.go +++ b/core/internal/consumer/maintenance_test.go @@ -23,23 +23,59 @@ import ( "testing" "github.com/cloudnative-pg/klio/core/internal/client/klioclient" + "github.com/cloudnative-pg/klio/core/internal/kopia" ) +func snapshot(path, backup, content, tablespace string) kopia.Manifest { + return kopia.Manifest{ + Source: kopia.SourceInfo{Host: "c", UserName: "klio", Path: path}, + Tags: map[string]string{ + klioclient.BackupNameTagName: backup, + klioclient.BackupContentTagName: content, + klioclient.TablespaceNameTagName: tablespace, + }, + } +} + func TestKeepUntilOnTier2(t *testing.T) { - tier2 := klioclient.BackupList{{Name: "backup-1"}, {Name: "backup-2"}} - keep := keepUntilOnTier2(tier2) + tier1 := []kopia.Manifest{ + snapshot("/pgdata", "b1", "pgdata", ""), + snapshot("/pgdata_meta", "b1", "metadata", ""), + snapshot("/tbs/a", "b1", "tablespace", "a"), + snapshot("/pgdata", "b2", "pgdata", ""), + snapshot("/pgdata_meta", "b2", "metadata", ""), + snapshot("/tbs/a", "b2", "tablespace", "a"), + snapshot("/pgdata", "b3", "pgdata", ""), + snapshot("/pgdata_meta", "b3", "metadata", ""), + } + // b1 fully relayed, b2 has its metadata but not its tablespace on tier2, + // b3 not relayed at all. + tier2 := []kopia.Manifest{ + snapshot("/pgdata", "b1", "pgdata", ""), + snapshot("/pgdata_meta", "b1", "metadata", ""), + snapshot("/tbs/a", "b1", "tablespace", "a"), + snapshot("/pgdata", "b2", "pgdata", ""), + snapshot("/pgdata_meta", "b2", "metadata", ""), + } + + keep := keepUntilOnTier2(tier1, tier2) - // A backup already on tier2 is deletable (not kept). - if keep("backup-1") { - t.Error("keepUntilOnTier2() kept backup-1, which is already on tier2") + if keep("b1") { + t.Error("kept b1, which is complete on tier2") + } + if !keep("b2") { + t.Error("did not keep b2, whose tablespace is missing on tier2") + } + if !keep("b3") { + t.Error("did not keep b3, which is not on tier2") } - // A backup not yet on tier2 must be kept (protected from tier1 deletion). - if !keep("backup-3") { - t.Error("keepUntilOnTier2() did not keep backup-3, which is not on tier2") + // A backup unknown to tier1 has nothing to protect. + if keep("b4") { + t.Error("kept b4, which has no tier1 snapshots") } - // With no tier2 backups, everything is kept. - if keepAll := keepUntilOnTier2(nil); !keepAll("backup-1") { - t.Error("keepUntilOnTier2(nil) did not keep backup-1") + // With no tier2 snapshots, everything on tier1 is kept. + if keepAll := keepUntilOnTier2(tier1, nil); !keepAll("b1") { + t.Error("keepUntilOnTier2(tier1, nil) did not keep b1") } } From f0b48e072c3359afab2e213b0df4cac72a7d93a7 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:04:44 +0200 Subject: [PATCH 14/23] fix(retention): fail the backup listing when a metadata snapshot is unreadable ListBackups skipped a backup whose metadata could not be read and returned the rest as if the catalog were complete. The catalog now drives base backup retention and WAL cleanup, so a transient read error made them delete one backup, or the WALs of one backup, too many. A metadata read error now fails the listing, and the callers fail closed. Signed-off-by: Armando Ruocco --- core/internal/client/klioclient/kopia/list.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/internal/client/klioclient/kopia/list.go b/core/internal/client/klioclient/kopia/list.go index 82f1557d..031c8375 100644 --- a/core/internal/client/klioclient/kopia/list.go +++ b/core/internal/client/klioclient/kopia/list.go @@ -53,7 +53,10 @@ func (s *Connection) GetMetadata( return nil, newNoBackupFoundError(hostname, name) } -// ListBackups list all the backups in the repository. +// ListBackups list all the backups in the repository. A metadata snapshot +// that cannot be read fails the whole listing: the catalog drives retention +// and WAL cleanup, and silently dropping a backup from it would let them +// delete one backup too many. func (s *Connection) ListBackups(ctx context.Context, hostname string) (klioclient.BackupList, error) { contextLogger := log.FromContext(ctx) @@ -72,10 +75,10 @@ func (s *Connection) ListBackups(ctx context.Context, hostname string) (klioclie metadata, err := s.restoreMetadata(ctx, entry.ID) if err != nil { - contextLogger.Error(err, "Error while decoding backup metadata, skipping", "id", entry.ID) - } else { - result = append(result, *metadata) + return nil, fmt.Errorf("while reading the metadata of backup snapshot %q: %w", entry.ID, err) } + + result = append(result, *metadata) } return result, nil From d8329bb471bc4251e11bd9cb67ed89de13f72356 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:04:44 +0200 Subject: [PATCH 15/23] fix(retention): bind ApplyRetention to the client certificate cluster The WAL server requires a client certificate but never bound its Common Name to the cluster named in the request, so any certificate signed by the CA could apply a retention policy to another cluster's backups. The host part of the userName@hostName Common Name must now match the cluster in the ApplyRetention request; the other calls are unchanged. Signed-off-by: Armando Ruocco --- core/internal/server/walserver/backup.go | 7 ++ core/internal/server/walserver/identity.go | 59 +++++++++++++++ .../server/walserver/identity_test.go | 75 +++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 core/internal/server/walserver/identity.go create mode 100644 core/internal/server/walserver/identity_test.go diff --git a/core/internal/server/walserver/backup.go b/core/internal/server/walserver/backup.go index 617340d5..1b1ba323 100644 --- a/core/internal/server/walserver/backup.go +++ b/core/internal/server/walserver/backup.go @@ -93,6 +93,13 @@ func (w *Implementation) ApplyRetention( return nil, status.Errorf(codes.InvalidArgument, "invalid cluster name: %v", err) } + // Retention deletes backups, so the caller must prove it owns the cluster: + // the host part of the client certificate Common Name (userName@hostName) + // must match the requested cluster. + if err := checkPeerCluster(ctx, request.GetClusterName()); err != nil { + return nil, err + } + tier1Policy := parseRetentionPolicy(ctx, "tier1", request.GetTier1RetentionPolicy()) tier2Policy := parseRetentionPolicy(ctx, "tier2", request.GetTier2RetentionPolicy()) diff --git a/core/internal/server/walserver/identity.go b/core/internal/server/walserver/identity.go new file mode 100644 index 00000000..4e50fc1f --- /dev/null +++ b/core/internal/server/walserver/identity.go @@ -0,0 +1,59 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package walserver + +import ( + "context" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// checkPeerCluster verifies that the client certificate of the caller was +// issued for clusterName. The Common Name has the form userName@hostName, +// where the host part is the cluster the certificate grants access to. +func checkPeerCluster(ctx context.Context, clusterName string) error { + p, ok := peer.FromContext(ctx) + if !ok { + return status.Error(codes.Unauthenticated, "no peer information") + } + + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(tlsInfo.State.PeerCertificates) == 0 { + return status.Error(codes.Unauthenticated, "no client certificate") + } + + commonName := tlsInfo.State.PeerCertificates[0].Subject.CommonName + _, certCluster, found := strings.Cut(commonName, "@") + if !found || certCluster == "" { + return status.Errorf(codes.PermissionDenied, + "client certificate Common Name %q is not in the form userName@hostName", commonName) + } + + if certCluster != clusterName { + return status.Errorf(codes.PermissionDenied, + "client certificate is issued for cluster %q, not %q", certCluster, clusterName) + } + + return nil +} diff --git a/core/internal/server/walserver/identity_test.go b/core/internal/server/walserver/identity_test.go new file mode 100644 index 00000000..b019dd17 --- /dev/null +++ b/core/internal/server/walserver/identity_test.go @@ -0,0 +1,75 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package walserver + +import ( + "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func peerContext(commonName string) func() context.Context { + return func() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{Subject: pkix.Name{CommonName: commonName}}}, + }}, + }) + } +} + +func TestCheckPeerCluster(t *testing.T) { + tests := []struct { + name string + ctx func() context.Context + cluster string + wantCode codes.Code + }{ + {name: "matching cluster", ctx: peerContext("klio@cluster-a"), cluster: "cluster-a", wantCode: codes.OK}, + { + name: "other cluster", ctx: peerContext("klio@cluster-a"), cluster: "cluster-b", + wantCode: codes.PermissionDenied, + }, + {name: "malformed CN", ctx: peerContext("cluster-a"), cluster: "cluster-a", wantCode: codes.PermissionDenied}, + {name: "empty host", ctx: peerContext("klio@"), cluster: "", wantCode: codes.PermissionDenied}, + {name: "no peer", ctx: context.Background, cluster: "cluster-a", wantCode: codes.Unauthenticated}, + { + name: "no certificate", cluster: "cluster-a", wantCode: codes.Unauthenticated, + ctx: func() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := status.Code(checkPeerCluster(tt.ctx(), tt.cluster)); got != tt.wantCode { + t.Fatalf("checkPeerCluster() code = %v, want %v", got, tt.wantCode) + } + }) + } +} From 266c45507ff01237f9468e249004425f00afa64e Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:06:10 +0200 Subject: [PATCH 16/23] fix(retention): do not wait for tier2 on backups that skip the relay With tier2 enabled, the tier1 retention guard protected every backup that had not reached tier2, including backups taken with tier2 backup disabled, which never will. Those backups were never deleted from tier1. The client now records in the backup metadata whether it asked for the tier2 relay, and the guard lets tier1 retention delete a backup that was never meant to reach tier2. Backups taken by older clients carry no annotation and keep the protective behaviour. Signed-off-by: Armando Ruocco --- core/cmd/backup/run.go | 1 + core/internal/client/klioclient/backup.go | 14 +++++++++++++- core/internal/client/klioclient/consts.go | 13 +++++++++++++ core/internal/consumer/backup.go | 6 ++++-- core/internal/consumer/maintenance.go | 18 +++++++++++++++--- core/internal/consumer/maintenance_test.go | 19 ++++++++++++++----- core/internal/consumer/retention_test.go | 4 ++-- 7 files changed, 62 insertions(+), 13 deletions(-) diff --git a/core/cmd/backup/run.go b/core/cmd/backup/run.go index 7971a138..f24ed6ff 100644 --- a/core/cmd/backup/run.go +++ b/core/cmd/backup/run.go @@ -119,6 +119,7 @@ func runBackup(cmd *cobra.Command, _ []string) error { backupName, _ := cmd.Flags().GetString("name") opts.Name = backupName + opts.SendToTier2 = tier2 if err := backupExecutor.Start(cmd.Context(), opts); err != nil { return cli.NewCodedError( diff --git a/core/internal/client/klioclient/backup.go b/core/internal/client/klioclient/backup.go index 77a6f1a8..d3a64998 100644 --- a/core/internal/client/klioclient/backup.go +++ b/core/internal/client/klioclient/backup.go @@ -47,7 +47,8 @@ type BackupExecutor struct { uploader Client - startedAt int64 + startedAt int64 + sendToTier2 bool } // NewBackupExecutor creates a new backup executor for the passed implementation. @@ -65,6 +66,10 @@ type BackupOptions struct { // Name is the backup name. If not set a new name will be generated // using the current timestamp. Name string + + // SendToTier2 records in the backup metadata whether the backup is meant + // to be relayed to tier2. + SendToTier2 bool } // Start starts the execution of a backup. @@ -76,6 +81,7 @@ func (b *BackupExecutor) Start(ctx context.Context, opts BackupOptions) error { if opts.Name != "" { b.name = opts.Name } + b.sendToTier2 = opts.SendToTier2 row := b.Connection.QueryRow(ctx, "SHOW data_directory") if err := row.Scan(&b.pgData); err != nil { @@ -216,6 +222,12 @@ func (b *BackupExecutor) Close(ctx context.Context) (*BackupMetadata, error) { } } + relay := Tier2RelaySkipped + if b.sendToTier2 { + relay = Tier2RelayRequested + } + metadata.SetAnnotation(Tier2RelayAnnotationName, relay) + contextLogger.Info("Uploading backup metadata") if err := b.uploader.UploadBackupMetadata(ctx, b.name, metadata); err != nil { return nil, fmt.Errorf("while uploading backup metadata: %w", err) diff --git a/core/internal/client/klioclient/consts.go b/core/internal/client/klioclient/consts.go index 20f8170a..5eed62cb 100644 --- a/core/internal/client/klioclient/consts.go +++ b/core/internal/client/klioclient/consts.go @@ -30,3 +30,16 @@ const BackupContentTagName = "klio.io/content" // TablespaceNameTagName is the name of the tag containing the // name of the tablespace. const TablespaceNameTagName = "klio.io/tablespaceName" + +// Tier2RelayAnnotationName is the metadata annotation recording whether the +// client asked for the backup to be relayed to tier2. Backups taken before +// the annotation existed carry none and are treated as relayed. +const Tier2RelayAnnotationName = "klio.io/tier2-relay" + +// Tier2RelaySkipped is the Tier2RelayAnnotationName value of a backup that is +// not meant to reach tier2, so tier1 retention need not wait for it. +const Tier2RelaySkipped = "skipped" + +// Tier2RelayRequested is the Tier2RelayAnnotationName value of a backup that +// must reach tier2 before tier1 retention can delete it. +const Tier2RelayRequested = "requested" diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index 26c3e761..c5006edb 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -347,7 +347,7 @@ func (d *Backup) applyRetention( client retentionClient, clusterName string, policy retention.Policy, - keep func(name string) bool, + keep func(backup *klioclient.BackupMetadata) bool, ) error { contextLogger := log.FromContext(ctx) @@ -362,8 +362,10 @@ func (d *Backup) applyRetention( return fmt.Errorf("while listing backups for cluster %q: %w", clusterName, err) } + byName := make(map[string]*klioclient.BackupMetadata, len(backups)) catalog := make([]retention.Backup, len(backups)) for i := range backups { + byName[backups[i].Name] = &backups[i] catalog[i] = retention.Backup{ Name: backups[i].Name, StartedAt: backups[i].StartedAt, @@ -373,7 +375,7 @@ func (d *Backup) applyRetention( var errs error for _, expired := range retention.Evaluate(catalog, policy) { - if keep != nil && keep(expired.Name) { + if keep != nil && keep(byName[expired.Name]) { contextLogger.Info("Retention expired a backup that is not deletable yet, keeping it", "cluster", clusterName, "backup", expired.Name) diff --git a/core/internal/consumer/maintenance.go b/core/internal/consumer/maintenance.go index 92dcc52e..c7e23059 100644 --- a/core/internal/consumer/maintenance.go +++ b/core/internal/consumer/maintenance.go @@ -76,7 +76,10 @@ func (d *Backup) runTier1Retention(ctx context.Context, task *queue.BackupTask) // tier1RetentionGuard returns a predicate that reports whether a tier1 backup // must be kept because it has not yet been fully migrated to tier2. When tier2 // is not configured there is nothing to protect and the predicate is nil. -func (d *Backup) tier1RetentionGuard(ctx context.Context, clusterName string) (func(name string) bool, error) { +func (d *Backup) tier1RetentionGuard( + ctx context.Context, + clusterName string, +) (func(backup *klioclient.BackupMetadata) bool, error) { if !d.tier2Enabled { return nil, nil } @@ -101,7 +104,10 @@ func (d *Backup) tier1RetentionGuard(ctx context.Context, clusterName string) (f // snapshot migration with no ordering between a backup's parts, so the tier2 // metadata snapshot alone does not prove the data is there: a backup is // deletable only when every tier1 snapshot of it has a counterpart on tier2. -func keepUntilOnTier2(tier1Snapshots, tier2Snapshots []kopia.Manifest) func(name string) bool { +// A backup the client never meant to relay has nothing to wait for. +func keepUntilOnTier2( + tier1Snapshots, tier2Snapshots []kopia.Manifest, +) func(backup *klioclient.BackupMetadata) bool { onTier2 := stringset.New() for i := range tier2Snapshots { onTier2.Put(snapshotPartKey(tier2Snapshots[i])) @@ -119,7 +125,13 @@ func keepUntilOnTier2(tier1Snapshots, tier2Snapshots []kopia.Manifest) func(name } } - return incomplete.Has + return func(backup *klioclient.BackupMetadata) bool { + if backup.Annotations[klioclient.Tier2RelayAnnotationName] == klioclient.Tier2RelaySkipped { + return false + } + + return incomplete.Has(backup.Name) + } } // snapshotPartKey identifies one part of a backup (pgdata, metadata, control diff --git a/core/internal/consumer/maintenance_test.go b/core/internal/consumer/maintenance_test.go index 49460292..797f1d84 100644 --- a/core/internal/consumer/maintenance_test.go +++ b/core/internal/consumer/maintenance_test.go @@ -59,22 +59,31 @@ func TestKeepUntilOnTier2(t *testing.T) { } keep := keepUntilOnTier2(tier1, tier2) + backup := func(name string) *klioclient.BackupMetadata { + return &klioclient.BackupMetadata{Name: name} + } - if keep("b1") { + if keep(backup("b1")) { t.Error("kept b1, which is complete on tier2") } - if !keep("b2") { + if !keep(backup("b2")) { t.Error("did not keep b2, whose tablespace is missing on tier2") } - if !keep("b3") { + if !keep(backup("b3")) { t.Error("did not keep b3, which is not on tier2") } // A backup unknown to tier1 has nothing to protect. - if keep("b4") { + if keep(backup("b4")) { t.Error("kept b4, which has no tier1 snapshots") } + // A backup the client never meant to relay is not waited for. + skipped := backup("b3") + skipped.SetAnnotation(klioclient.Tier2RelayAnnotationName, klioclient.Tier2RelaySkipped) + if keep(skipped) { + t.Error("kept b3, which was never meant to reach tier2") + } // With no tier2 snapshots, everything on tier1 is kept. - if keepAll := keepUntilOnTier2(tier1, nil); !keepAll("b1") { + if keepAll := keepUntilOnTier2(tier1, nil); !keepAll(backup("b1")) { t.Error("keepUntilOnTier2(tier1, nil) did not keep b1") } } diff --git a/core/internal/consumer/retention_test.go b/core/internal/consumer/retention_test.go index 22ff186c..a121a8db 100644 --- a/core/internal/consumer/retention_test.go +++ b/core/internal/consumer/retention_test.go @@ -62,7 +62,7 @@ func TestApplyRetention(t *testing.T) { name string client *fakeRetentionClient policy retention.Policy - keep func(string) bool + keep func(*klioclient.BackupMetadata) bool wantDeleted []string wantErr error }{ @@ -80,7 +80,7 @@ func TestApplyRetention(t *testing.T) { name: "keep predicate protects an expired backup", client: &fakeRetentionClient{backups: catalog}, policy: retention.Policy{Latest: 1}, - keep: func(name string) bool { return name == "b1" }, + keep: func(b *klioclient.BackupMetadata) bool { return b.Name == "b1" }, wantDeleted: []string{"b2"}, }, { From c04759060dbdbdab6e22491cc3b290ee7358f08e Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 12:06:10 +0200 Subject: [PATCH 17/23] docs(retention): document backup ordering and the tier2 guard Retention orders backups by the start time taken from the instance clock, so a clock skew after a switchover can misorder them. Spell that out, and describe when tier 1 retention waits for tier 2. Signed-off-by: Armando Ruocco --- documentation/web/docs/user/plugin_configuration.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/documentation/web/docs/user/plugin_configuration.md b/documentation/web/docs/user/plugin_configuration.md index e7d1d2b6..3df57bc9 100644 --- a/documentation/web/docs/user/plugin_configuration.md +++ b/documentation/web/docs/user/plugin_configuration.md @@ -243,6 +243,16 @@ for recovery by any retained backup. The retention policy is optional and must be set to at least `1` when present. Omit it entirely to keep every backup. +Backups are ordered by the start time recorded by the PostgreSQL instance +that took them. Keep the clocks of the instances in sync (as Kubernetes nodes +normally are): after a switchover, a clock behind the previous primary's makes +the newest backup look older than it is, and a tight `latest` policy may +expire it first. + +With tier 2 enabled, a tier 1 backup is never deleted before all of its +snapshots have reached tier 2, unless it was taken with tier 2 backup +disabled. + A change to the retention policy takes effect the next time a backup is taken. To apply it immediately, for example to reclaim space after tightening the policy, run `klio retention apply`. From 6fb8741782a9d670d4ed0d3291b17b8db8641264 Mon Sep 17 00:00:00 2001 From: Gabriele Quaresima Date: Mon, 14 Sep 2026 13:57:38 +0200 Subject: [PATCH 18/23] test(retention): verify tier1 retention and share the flow with tier2 Add a tier1-only e2e that mirrors the tier2 retention test: configure a tier1 retention policy of latest:2, take three backups, and assert the retention manager keeps exactly two and deletes the oldest, then tighten the policy to one and verify `klio retention apply` leaves only the newest. Both retention features run the same tier-parameterized flow (verifyRetentionAndOnDemandApply): the tier1 and tier2 Run methods differ only in the tier annotation, the policy the on-demand step edits, and the tier2-only WAL and recovery-gate checks layered on top. Generalize the tier2 retention helpers accordingly, add a Tier1RetentionPolicy option to the Klio plugin-configuration template, register the new Tier1Retention feature, and update the e2e test-structure docs. Assisted-by: Claude Signed-off-by: Gabriele Quaresima --- .../web/docs/developer/running-e2e-tests.md | 8 +- operator/test/e2e/main_test.go | 1 + operator/test/e2e/tier1_retention_test.go | 124 ++++++++++++ operator/test/klio/features/retention_flow.go | 175 +++++++++++++++++ .../test/klio/features/tier1_retention.go | 182 +++++++++++++++++ .../test/klio/features/tier2_retention.go | 183 +++++------------- operator/test/utils/templates/klio/klio.go | 9 + 7 files changed, 550 insertions(+), 132 deletions(-) create mode 100644 operator/test/e2e/tier1_retention_test.go create mode 100644 operator/test/klio/features/retention_flow.go create mode 100644 operator/test/klio/features/tier1_retention.go diff --git a/documentation/web/docs/developer/running-e2e-tests.md b/documentation/web/docs/developer/running-e2e-tests.md index 9c7edc40..946db107 100644 --- a/documentation/web/docs/developer/running-e2e-tests.md +++ b/documentation/web/docs/developer/running-e2e-tests.md @@ -129,8 +129,14 @@ The E2E tests are located in `operator/test/e2e/` and include: by both of the above; also asserts that the read-only (tier2-only) recovery Server gets the unified `klio` PVC/mount, same as a tier1 server +- **`tier1_retention_test.go`** - Backup retention policy enforcement on a + tier1-only deployment: takes more backups than the policy keeps, verifies + the retention manager deletes the oldest, then tightens the policy and + verifies `klio retention apply` prunes on demand (`Tier1Retention`) - **`tier2_retention_test.go`** - Backup and WAL retention policy - enforcement in tier2 storage (`Tier2Retention`) + enforcement in tier2 storage: shares the tier1 automatic and on-demand + retention flow and adds tier2-only WAL retention and recovery-gate + checks (`Tier2Retention`) - **`compression_test.go`** - Kopia compression policies: verifies the repository-wide policy set on the Server applies globally and that the per-cluster policy set on the PluginConfiguration overrides it, by diff --git a/operator/test/e2e/main_test.go b/operator/test/e2e/main_test.go index b5a80ff2..8e71debb 100644 --- a/operator/test/e2e/main_test.go +++ b/operator/test/e2e/main_test.go @@ -61,6 +61,7 @@ func TestMain(m *testing.M) { runner.RegisterFeature(RecoverClusterFromTier2(envconf.RandomName("recovery-from-tier2", 32))) runner.RegisterFeature(RecoverClusterFromTier2Pitr(envconf.RandomName("recovery-from-tier2-pitr", 32))) runner.RegisterFeature(PluginConfigurationUpdate(envconf.RandomName("plugin-config-update", 32))) + runner.RegisterFeature(Tier1Retention(envconf.RandomName("tier1-retention", 32))) runner.RegisterFeature(Tier2Retention(envconf.RandomName("tier2-retention", 32))) runner.RegisterFeature(Compression(envconf.RandomName("compression", 32))) runner.RegisterFeature(WALRetentionQueueAwareness(envconf.RandomName("wal-retention-queue", 32))) diff --git a/operator/test/e2e/tier1_retention_test.go b/operator/test/e2e/tier1_retention_test.go new file mode 100644 index 00000000..2a7600a7 --- /dev/null +++ b/operator/test/e2e/tier1_retention_test.go @@ -0,0 +1,124 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package e2e + +import ( + "fmt" + + cnpgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" + klioFeatures "github.com/cloudnative-pg/klio/operator/test/klio/features" + "github.com/cloudnative-pg/klio/operator/test/utils/templates/certificates" + "github.com/cloudnative-pg/klio/operator/test/utils/templates/cnpg" + "github.com/cloudnative-pg/klio/operator/test/utils/templates/klio" + "github.com/cloudnative-pg/klio/operator/test/utils/templates/secrets" +) + +// Tier1Retention returns a Tier1RetentionFeature for a tier1-only deployment. +// It keeps the two most recent backups, takes three, and verifies the retention +// manager deleted the oldest, then tightens the policy to one and verifies +// `klio retention apply` leaves only the newest. This exercises the tier1 +// retention path, which is distinct from the tier2 one covered by +// Tier2Retention. +func Tier1Retention(namespace string) *klioFeatures.Tier1RetentionFeature { + const ( + cnpgClusterName = "pg-tier1-retention" + pluginConfigurationName = "klio-plugin-configuration" + keepLatest = 2 + ) + + namespaceObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + + issuer := certificates.GetSelfSignedIssuerObject("selfsigned-issuer", namespace) + certificate := certificates.GetCertificateObject("test", namespace, []string{klioServerName}, issuer) + caCertificate := certificates.GetCACertificateObject("test-ca", namespace, issuer) + caIssuer := certificates.GetCAIssuerObject("test-ca-issuer", namespace, caCertificate.Spec.SecretName) + userCertificate := certificates.GetUserCertificateObject( + "klio-user", namespace, "klio-user@"+cnpgClusterName, caIssuer) + + cnpgCluster := cnpg.GetCnpgClusterObject(cnpgClusterName, namespace, 1, pluginConfigurationName, + cnpg.ClusterTemplateOptions{StorageClass: testCfg.StorageClass}) + + klioPluginConfiguration := klio.GetPluginConfigurationObject( + pluginConfigurationName, + namespace, + klio.PluginConfigurationTemplateOptions{ + ServerCertificate: certificate, + ClientCertificate: userCertificate, + ClusterName: cnpgClusterName, + Tier1RetentionPolicy: &kliov1alpha1.RetentionPolicy{Latest: keepLatest}, + }, + ) + + ageSecrets := secrets.GetKlioAgeEncryptionSecrets("encryption", namespace, "testencryptionpassword123") + klioServer := klio.GetServerObject( + klioServerName, + namespace, + klio.ServerTemplateOptions{ + Image: testCfg.ServerImage, + StorageClass: testCfg.StorageClass, + TLSSecretName: certificate.Spec.SecretName, + ClientCASecretName: caCertificate.Spec.SecretName, + Encryption: klio.EncryptionOptions{ + EncryptionKeySecretName: ageSecrets.EncryptionKeySecret.Name, + EncryptionKeyFileName: "encryption-key.age", + IdentitySecretName: ageSecrets.IdentitySecret.Name, + IdentityFileName: "identity.txt", + }, + }, + ) + + // Take one more backup than the policy keeps, so retention must delete one. + backups := make([]*cnpgv1.Backup, 0, keepLatest+1) + for i := range keepLatest + 1 { + backups = append(backups, cnpg.GetCnpgBackupObject( + fmt.Sprintf("test-backup-%d", i+1), namespace, cnpgv1.BackupTargetPrimary, cnpgCluster)) + } + + scenario := commonBackupRestoreScenario{ + namespace: namespaceObj, + cnpgCluster: cnpgCluster, + userCertificate: userCertificate, + encryptionSecret: ageSecrets.EncryptionKeySecret, + identitySecret: ageSecrets.IdentitySecret, + issuer: issuer, + caIssuer: caIssuer, + caCertificate: caCertificate, + certificate: certificate, + klioServer: klioServer, + klioPluginConfigurationSource: klioPluginConfiguration, + name: "Tier1Retention", + } + + return klioFeatures.NewTier1RetentionFeature(klioFeatures.Tier1RetentionFeatureConfig{ + Name: "Tier1Retention", + Setup: scenario.Setup, + Teardown: scenario.Teardown, + Backups: backups, + KlioServer: klioServer, + Namespace: namespace, + KeepLatest: keepLatest, + ClusterName: cnpgClusterName, + PluginConfigurationName: pluginConfigurationName, + }) +} diff --git a/operator/test/klio/features/retention_flow.go b/operator/test/klio/features/retention_flow.go new file mode 100644 index 00000000..4b4324a8 --- /dev/null +++ b/operator/test/klio/features/retention_flow.go @@ -0,0 +1,175 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package features + +import ( + "context" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + "github.com/stretchr/testify/require" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + + machineryConditions "github.com/cloudnative-pg/klio/operator/test/machinery/pkg/conditions" +) + +// retentionFlowParams carries what the tier-agnostic retention flow needs to +// exercise one tier's retention: automatic deletion of the oldest backup once +// newer backups push it outside the policy, followed by on-demand deletion via +// `klio retention apply`. Tier1 and tier2 differ only in the tier annotation, +// the label used in logs, and which tier's policy setRetentionLatest edits. +type retentionFlowParams struct { + backups []*cnpgv1.Backup + serverName string + namespace string + clusterName string + keepLatest int + tierLabel string + tierAnnotation string + pluginConfigurationName string + backupTimeout time.Duration + retentionTimeout time.Duration + checkInterval time.Duration + // setRetentionLatest tightens this tier's policy to keep `latest` backups. + setRetentionLatest func(ctx context.Context, t *testing.T, r *resources.Resources, latest int) + // onFirstBackup, if set, runs once right after the first backup reaches the + // tier. Tier2 uses it to record a WAL directory baseline. + onFirstBackup func(ctx context.Context) +} + +// verifyRetentionAndOnDemandApply runs the tier-agnostic retention flow shared +// by the tier1 and tier2 retention features. It creates more backups than the +// policy keeps and verifies the tier ends up with exactly keepLatest backups +// with the oldest deleted, then tightens the policy to a single backup, applies +// it on demand with `klio retention apply`, and verifies only the newest +// remains. This exercises the full server-side path: PluginConfiguration CR -> +// operator -> klio-plugin config -> CloseBackup GRPC -> NATS queue -> backup +// consumer -> Klio-managed retention. +func verifyRetentionAndOnDemandApply( + ctx context.Context, + t *testing.T, + r *resources.Resources, + p retentionFlowParams, +) { + t.Helper() + + // Name of the first (oldest) backup. Retention must delete it once newer + // backups push it outside the policy. + var oldestBackupName string + + for i, backup := range p.backups { + t.Logf("Creating backup %d/%d: %s", i+1, len(p.backups), backup.Name) + require.NoError(t, r.Create(ctx, backup), "failed to create backup %s", backup.Name) + + err := wait.For( + machineryConditions.BackupIsCompleted(r, backup), + wait.WithTimeout(p.backupTimeout), + wait.WithInterval(p.checkInterval), + ) + require.NoError(t, err, "backup %s did not complete", backup.Name) + t.Logf("Backup %s completed successfully", backup.Name) + + // After more backups than the policy keeps, the older ones are deleted. + expectedBackups := min(i+1, p.keepLatest) + err = wait.For( + checkTierHasBackups(r, p.namespace, p.serverName, p.tierAnnotation, expectedBackups), + wait.WithTimeout(p.retentionTimeout), + wait.WithInterval(p.checkInterval), + ) + require.NoError(t, err, "%s retention not applied after backup %d", p.tierLabel, i+1) + t.Logf("%s has expected %d backup(s) after backup %d", p.tierLabel, expectedBackups, i+1) + + if i == 0 { + names, listErr := listTierBackupNames( + ctx, r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation) + require.NoError(t, listErr, "failed to list %s backups after the first backup", p.tierLabel) + require.Len(t, names, 1, "expected exactly one %s backup after the first backup", p.tierLabel) + oldestBackupName = names[0] + t.Logf("Oldest %s backup recorded: %s", p.tierLabel, oldestBackupName) + + if p.onFirstBackup != nil { + p.onFirstBackup(ctx) + } + } + } + + // Automatic retention: exactly keepLatest backups remain and the oldest was + // the one the retention manager deleted (the newest survive). + t.Logf("Retention verification: %s should have exactly %d backup(s)", p.tierLabel, p.keepLatest) + err := wait.For( + checkTierHasBackups(r, p.namespace, p.serverName, p.tierAnnotation, p.keepLatest), + wait.WithTimeout(p.retentionTimeout), + wait.WithInterval(p.checkInterval), + ) + require.NoError(t, err, "%s backup count verification failed", p.tierLabel) + + survivingNames, err := listTierBackupNames( + ctx, r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation) + require.NoError(t, err, "could not list surviving %s backups", p.tierLabel) + require.NotContains(t, survivingNames, oldestBackupName, + "the oldest backup should have been deleted by the %s retention manager", p.tierLabel) + t.Logf("PASSED: %s has exactly %d backup(s) and the oldest (%s) was deleted", + p.tierLabel, p.keepLatest, oldestBackupName) + + // On-demand retention: tighten the policy to keep a single backup and apply + // it immediately with `klio retention apply`, without taking a new backup. + // Only the newest backup must remain afterwards. + newestBackupName := survivingNames[0] + t.Logf("On-demand retention: tightening %s retention to 1 and running `klio retention apply`", p.tierLabel) + p.setRetentionLatest(ctx, t, r, 1) + + instancePodName := p.backups[len(p.backups)-1].Status.InstanceID.PodName + require.NotEmpty(t, instancePodName, "backup instance pod name should be set") + + // The `klio retention apply` command sends the policy from the pod's mounted + // config, which the operator updates asynchronously after the + // PluginConfiguration change. Re-running the (idempotent) command until + // exactly one backup remains absorbs both the config propagation and the + // asynchronous consumer processing. + err = wait.For( + func(ctx context.Context) (bool, error) { + if applyErr := runRetentionApply(ctx, r, p.namespace, instancePodName); applyErr != nil { + t.Logf("klio retention apply not ready yet: %v", applyErr) + + return false, nil + } + + names, listErr := listTierBackupNames( + ctx, r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation) + if listErr != nil { + return false, nil //nolint:nilerr + } + + return len(names) == 1, nil + }, + wait.WithTimeout(p.retentionTimeout), + wait.WithInterval(p.checkInterval), + ) + require.NoError(t, err, "on-demand %s retention did not converge to a single backup", p.tierLabel) + + finalNames, err := listTierBackupNames( + ctx, r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation) + require.NoError(t, err, "could not list surviving %s backups", p.tierLabel) + require.Equal(t, []string{newestBackupName}, finalNames, + "only the newest backup should remain after `klio retention apply`") + t.Logf("PASSED: only the newest %s backup (%s) remains after apply", p.tierLabel, newestBackupName) +} diff --git a/operator/test/klio/features/tier1_retention.go b/operator/test/klio/features/tier1_retention.go new file mode 100644 index 00000000..d76466c0 --- /dev/null +++ b/operator/test/klio/features/tier1_retention.go @@ -0,0 +1,182 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package features + +import ( + "context" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + "github.com/stretchr/testify/require" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/types" + + kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" +) + +// Tier1RetentionFeature defines a feature for testing tier1 backup retention on +// a tier1-only deployment. +type Tier1RetentionFeature struct { + name string + setup types.StepFunc + teardown types.StepFunc + backups []*cnpgv1.Backup + klioServer *kliov1alpha1.Server + namespace string + keepLatest int + backupTimeout time.Duration + checkInterval time.Duration + clusterName string + pluginConfigurationName string +} + +// Tier1RetentionFeatureConfig holds the configuration for creating a tier1 +// retention feature test. +type Tier1RetentionFeatureConfig struct { + // Name of the tier1 retention feature test. + Name string + // Setup function to initialize test resources. + Setup types.StepFunc + // Teardown function to clean up test resources. + Teardown types.StepFunc + // Backups are the backup resources to be created, in order. + Backups []*cnpgv1.Backup + // KlioServer is the Klio server resource. + KlioServer *kliov1alpha1.Server + // Namespace is the namespace where resources are created. + Namespace string + // KeepLatest is the number of backups the tier1 policy keeps. + KeepLatest int + // ClusterName is the name of the CNPG cluster whose backups are counted. + ClusterName string + // PluginConfigurationName is the name of the PluginConfiguration, used to + // tighten the retention policy for the on-demand `klio retention apply` step. + PluginConfigurationName string + // BackupTimeout is the timeout for each backup and its retention (defaults + // to 5 minutes). + BackupTimeout time.Duration + // CheckInterval is the interval for polling status (defaults to 10 seconds). + CheckInterval time.Duration +} + +// NewTier1RetentionFeature creates a new Tier1RetentionFeature with the given +// configuration. +func NewTier1RetentionFeature(config Tier1RetentionFeatureConfig) *Tier1RetentionFeature { + if config.BackupTimeout <= 0 { + config.BackupTimeout = 5 * time.Minute + } + if config.CheckInterval <= 0 { + config.CheckInterval = 10 * time.Second + } + + return &Tier1RetentionFeature{ + name: config.Name, + setup: config.Setup, + teardown: config.Teardown, + backups: config.Backups, + klioServer: config.KlioServer, + namespace: config.Namespace, + keepLatest: config.KeepLatest, + backupTimeout: config.BackupTimeout, + checkInterval: config.CheckInterval, + clusterName: config.ClusterName, + pluginConfigurationName: config.PluginConfigurationName, + } +} + +// Name returns the name of the tier1 retention feature. +func (f *Tier1RetentionFeature) Name() string { + return f.name +} + +// Setup initializes the tier1 retention feature test. +func (f *Tier1RetentionFeature) Setup() types.StepFunc { + return f.setup +} + +// Run executes the tier1 retention feature test. It creates more backups than +// the policy keeps and verifies that tier1 ends up with exactly `keepLatest` +// backups and that the oldest one was the backup the retention manager deleted +// (the newest survive). This exercises the full server-side tier1 path: +// PluginConfiguration CR -> operator -> klio-plugin config -> CloseBackup GRPC +// -> NATS queue -> backup consumer -> Klio-managed tier1 retention. It then +// tightens the policy to a single backup and applies it on demand with +// `klio retention apply`, verifying only the newest remains. +func (f *Tier1RetentionFeature) Run() types.StepFunc { + return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Helper() + t.Log("Running tier1 backup retention feature test") + + r, err := resources.New(cfg.Client().RESTConfig()) + require.NoError(t, err, "failed to create resources client") + + verifyRetentionAndOnDemandApply(ctx, t, r, retentionFlowParams{ + backups: f.backups, + serverName: f.klioServer.Name, + namespace: f.namespace, + clusterName: f.clusterName, + keepLatest: f.keepLatest, + tierLabel: "tier1", + tierAnnotation: tier1AnnotationName, + pluginConfigurationName: f.pluginConfigurationName, + backupTimeout: f.backupTimeout, + retentionTimeout: f.backupTimeout, + checkInterval: f.checkInterval, + setRetentionLatest: func(ctx context.Context, t *testing.T, r *resources.Resources, latest int) { + t.Helper() + updateTier1RetentionLatest(ctx, t, r, f.namespace, f.pluginConfigurationName, latest) + }, + }) + + return ctx + } +} + +// Teardown cleans up resources after the test is run. +func (f *Tier1RetentionFeature) Teardown() types.StepFunc { + return f.teardown +} + +// updateTier1RetentionLatest fetches the PluginConfiguration and sets its +// tier1 retention policy to keep the given number of most recent backups. +func updateTier1RetentionLatest( + ctx context.Context, + t *testing.T, + r *resources.Resources, + namespace string, + pluginConfigurationName string, + latest int, +) { + t.Helper() + + var pc kliov1alpha1.PluginConfiguration + require.NoError(t, r.Get(ctx, pluginConfigurationName, namespace, &pc), + "failed to get PluginConfiguration %q", pluginConfigurationName) + + require.NotNil(t, pc.Spec.Tier1, "PluginConfiguration should have a tier1 section") + if pc.Spec.Tier1.RetentionPolicy == nil { + pc.Spec.Tier1.RetentionPolicy = &kliov1alpha1.RetentionPolicy{} + } + pc.Spec.Tier1.RetentionPolicy.Latest = latest + + require.NoError(t, r.Update(ctx, &pc), "failed to update PluginConfiguration retention") +} diff --git a/operator/test/klio/features/tier2_retention.go b/operator/test/klio/features/tier2_retention.go index a82d9b98..8320a284 100644 --- a/operator/test/klio/features/tier2_retention.go +++ b/operator/test/klio/features/tier2_retention.go @@ -34,20 +34,20 @@ import ( "github.com/stretchr/testify/require" k8swait "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/e2e-framework/klient/k8s/resources" - "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/types" kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" "github.com/cloudnative-pg/klio/operator/internal/cnpgi" "github.com/cloudnative-pg/klio/operator/internal/klioconfig" - machineryConditions "github.com/cloudnative-pg/klio/operator/test/machinery/pkg/conditions" ) const ( serverContainerName = "server" // klioPodSuffix is the suffix added to the server name to form the pod name. klioPodSuffix = "-klio-0" + // tier1AnnotationName is the annotation key used to mark backups present in tier1. + tier1AnnotationName = "klio.io/tier1" // tier2AnnotationName is the annotation key used to mark backups present in tier2. tier2AnnotationName = "klio.io/tier2" // presentAnnotationValue is the value set when a backup is present in a tier. @@ -159,13 +159,12 @@ func (f *Tier2RetentionFeature) Setup() types.StepFunc { // This test validates the complete tier2 retention pipeline using a three-level // verification strategy: // -// 1. Retention Verification: creates more backups than `keepLatest` and verifies -// that tier2 ends up with exactly `keepLatest` backups and that the oldest one -// was the backup the retention manager deleted (the newest survive). This -// exercises the full path: PluginConfiguration CR -> operator -> klio-plugin -// config -> CloseBackup GRPC -> NATS queue -> backup consumer -> Klio-managed -// retention. It then tightens the policy to a single backup and applies it on -// demand with `klio retention apply`, verifying only the newest remains. +// 1. Retention Verification: runs the shared retention flow +// (verifyRetentionAndOnDemandApply): creates more backups than `keepLatest`, +// verifies tier2 ends up with exactly `keepLatest` backups with the oldest +// deleted, then tightens the policy to a single backup and applies it on +// demand with `klio retention apply`, verifying only the newest remains. An +// onFirstBackup hook records the WAL directory baseline used by Level 2. // // 2. WAL Retention Verification: Monitors WAL directory count before and after // retention to verify WAL cleanup is occurring. This is a soft check (logs @@ -187,120 +186,39 @@ func (f *Tier2RetentionFeature) Run() types.StepFunc { // Track WAL directory count to verify WAL retention (Level 2 verification) var walDirCountAfterFirstBackup int - // Name of the first (oldest) backup that reached tier2. Retention must - // eventually delete it once newer backups push it outside the policy. - var oldestTier2BackupName string - - // Create more backups than keepLatest to trigger retention - for i, backup := range f.backups { - t.Logf("Creating backup %d/%d: %s", i+1, len(f.backups), backup.Name) - require.NoError(t, r.Create(ctx, backup), "failed to create backup %s", backup.Name) - - // Wait for backup to complete - err = wait.For( - machineryConditions.BackupIsCompleted(r, backup), - wait.WithTimeout(f.backupTimeout), - wait.WithInterval(f.checkInterval), - ) - require.NoError(t, err, "backup %s did not complete", backup.Name) - t.Logf("Backup %s completed successfully", backup.Name) - - // Wait for backup to be replicated to tier2 - t.Logf("Waiting for backup %d to reach tier2...", i+1) - - // If we've taken more backups than retention allows, we expect the older ones to be deleted - expectedBackups := min(i+1, f.keepLatest) - - err = wait.For( - checkTier2HasBackups(r, f.namespace, f.klioServer.Name, expectedBackups), - wait.WithTimeout(f.replicationTimeout), - wait.WithInterval(f.checkInterval), - ) - require.NoError(t, err, "tier2 replication/retention not completed for backup %d", i+1) - t.Logf("Tier2 has expected %d backup(s) after backup %d", expectedBackups, i+1) - - // After the first backup, record the WAL directory count as baseline - // and remember which backup reached tier2 first (the oldest one). - if i == 0 { - names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) - require.NoError(t, listErr, "failed to list tier2 backups after the first backup") - require.Len(t, names, 1, "expected exactly one tier2 backup after the first backup") - oldestTier2BackupName = names[0] - t.Logf("Oldest tier2 backup recorded: %s", oldestTier2BackupName) - - walDirCountAfterFirstBackup, err = countTier2WALDirectories( - ctx, r, f.namespace, f.klioServer.Name, f.s3Prefix, f.clusterName) - if err != nil { - t.Logf("Warning: could not count WAL directories after first backup: %v", err) - } else { - t.Logf("WAL directories after first backup: %d", walDirCountAfterFirstBackup) - } - } - } - // ========================================== // Level 1: Retention Verification // ========================================== - // Verify that tier2 contains exactly keepLatest backups and that the - // oldest one was deleted by the retention manager (the newest survive). - t.Logf("[Level 1] Retention verification: tier2 should have exactly %d backup(s)", f.keepLatest) - err = wait.For( - checkTier2HasBackups(r, f.namespace, f.klioServer.Name, f.keepLatest), - wait.WithTimeout(f.replicationTimeout), - wait.WithInterval(f.checkInterval), - ) - require.NoError(t, err, "Level 1 failed: tier2 backup count verification failed") - - survivingNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) - require.NoError(t, err, "Level 1 failed: could not list surviving tier2 backups") - require.NotContains(t, survivingNames, oldestTier2BackupName, - "Level 1 failed: the oldest backup should have been deleted by the retention manager") - t.Logf("[Level 1] PASSED: tier2 has exactly %d backup(s) and the oldest (%s) was deleted", - f.keepLatest, oldestTier2BackupName) - - // ========================================== - // Level 1b: On-demand retention (klio retention apply) - // ========================================== - // Tighten the tier2 retention to keep a single backup and apply it - // immediately with `klio retention apply`, without taking a new backup. - // Only the newest backup must remain afterwards. - newestBackupName := survivingNames[0] - t.Log("[Level 1b] On-demand retention: tightening tier2 retention to 1 and running `klio retention apply`") - updateTier2RetentionLatest(ctx, t, r, f.namespace, f.pluginConfigurationName, 1) - - instancePodName := f.backups[len(f.backups)-1].Status.InstanceID.PodName - require.NotEmpty(t, instancePodName, "Level 1b failed: backup instance pod name should be set") - - // The `klio retention apply` command sends the policy from the pod's - // mounted config, which the operator updates asynchronously after the - // PluginConfiguration change. Re-running the (idempotent) command until - // exactly one backup remains absorbs both the config propagation and the - // asynchronous consumer processing. - err = wait.For( - func(ctx context.Context) (bool, error) { - if applyErr := runRetentionApply(ctx, r, f.namespace, instancePodName); applyErr != nil { - t.Logf("[Level 1b] klio retention apply not ready yet: %v", applyErr) - - return false, nil - } - - names, listErr := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) - if listErr != nil { - return false, nil //nolint:nilerr + // The shared flow verifies automatic retention (oldest deleted) and the + // on-demand `klio retention apply` (only the newest remains). The + // onFirstBackup hook records the WAL directory baseline for Level 2. + verifyRetentionAndOnDemandApply(ctx, t, r, retentionFlowParams{ + backups: f.backups, + serverName: f.klioServer.Name, + namespace: f.namespace, + clusterName: f.clusterName, + keepLatest: f.keepLatest, + tierLabel: "tier2", + tierAnnotation: tier2AnnotationName, + pluginConfigurationName: f.pluginConfigurationName, + backupTimeout: f.backupTimeout, + retentionTimeout: f.replicationTimeout, + checkInterval: f.checkInterval, + setRetentionLatest: func(ctx context.Context, t *testing.T, r *resources.Resources, latest int) { + t.Helper() + updateTier2RetentionLatest(ctx, t, r, f.namespace, f.pluginConfigurationName, latest) + }, + onFirstBackup: func(ctx context.Context) { + var walErr error + walDirCountAfterFirstBackup, walErr = countTier2WALDirectories( + ctx, r, f.namespace, f.klioServer.Name, f.s3Prefix, f.clusterName) + if walErr != nil { + t.Logf("Warning: could not count WAL directories after first backup: %v", walErr) + } else { + t.Logf("WAL directories after first backup: %d", walDirCountAfterFirstBackup) } - - return len(names) == 1, nil }, - wait.WithTimeout(f.replicationTimeout), - wait.WithInterval(f.checkInterval), - ) - require.NoError(t, err, "Level 1b failed: on-demand retention did not converge to a single backup") - - finalNames, err := listTier2BackupNames(ctx, r, f.namespace, f.klioServer.Name, f.clusterName) - require.NoError(t, err, "Level 1b failed: could not list surviving tier2 backups") - require.Equal(t, []string{newestBackupName}, finalNames, - "Level 1b failed: only the newest backup should remain after `klio retention apply`") - t.Logf("[Level 1b] PASSED: only the newest backup (%s) remains after apply", newestBackupName) + }) // ========================================== // Level 2: WAL Retention Verification (Soft Check) @@ -409,12 +327,14 @@ func verifyWALRetention( t.Log("[Level 2] PASSED: WAL directory growth is within acceptable bounds") } -// checkTier2HasBackups checks if tier2 has exactly the expected number of backups. -// Returns (false, nil) on transient errors to allow the wait to continue retrying. -func checkTier2HasBackups( +// checkTierHasBackups checks if the tier identified by tierAnnotation has exactly +// the expected number of backups. Returns (false, nil) on transient errors to +// allow the wait to continue retrying. +func checkTierHasBackups( r *resources.Resources, namespace string, serverName string, + tierAnnotation string, expectedCount int, ) k8swait.ConditionWithContextFunc { return func(ctx context.Context) (bool, error) { @@ -444,15 +364,15 @@ func checkTier2HasBackups( return false, nil //nolint:nilerr } - // Count backups present in tier2 (those with the tier2 annotation) - tier2Count := 0 + // Count backups present in the tier (those carrying the tier annotation) + count := 0 for i := range backups { - if backups[i].Annotations[tier2AnnotationName] == presentAnnotationValue { - tier2Count++ + if backups[i].Annotations[tierAnnotation] == presentAnnotationValue { + count++ } } - return tier2Count == expectedCount, nil + return count == expectedCount, nil } } @@ -502,15 +422,16 @@ func runRetentionApply( return nil } -// listTier2BackupNames returns the names of the backups currently present in -// tier2 (those carrying the tier2 annotation), ordered newest first by their -// start time. -func listTier2BackupNames( +// listTierBackupNames returns the names of the backups currently present in the +// tier identified by tierAnnotation for the given cluster, ordered newest first +// by their start time. +func listTierBackupNames( ctx context.Context, r *resources.Resources, namespace string, serverName string, clusterName string, + tierAnnotation string, ) ([]string, error) { podName := serverName + klioPodSuffix @@ -541,7 +462,7 @@ func listTier2BackupNames( if backups[i].ClusterName != clusterName { continue } - if backups[i].Annotations[tier2AnnotationName] == presentAnnotationValue { + if backups[i].Annotations[tierAnnotation] == presentAnnotationValue { names = append(names, backups[i].Name) } } diff --git a/operator/test/utils/templates/klio/klio.go b/operator/test/utils/templates/klio/klio.go index 01dab129..bee8c005 100644 --- a/operator/test/utils/templates/klio/klio.go +++ b/operator/test/utils/templates/klio/klio.go @@ -199,6 +199,8 @@ type PluginConfigurationTemplateOptions struct { EnableTier2Backup bool // EnableTier2Recovery enables tier2 recovery. EnableTier2Recovery bool + // Tier1RetentionPolicy is the retention policy for tier1. + Tier1RetentionPolicy *kliov1alpha1.RetentionPolicy // Tier2RetentionPolicy is the retention policy for tier2. Tier2RetentionPolicy *kliov1alpha1.RetentionPolicy } @@ -221,6 +223,13 @@ func GetPluginConfigurationObject( Mode: mode, } + // Only populate Tier1 when a retention policy is requested for it. + if opts.Tier1RetentionPolicy != nil { + spec.Tier1 = &kliov1alpha1.Tier1PluginConfiguration{ + RetentionPolicy: opts.Tier1RetentionPolicy, + } + } + // Only populate Tier2 if either backup or recovery is enabled if opts.EnableTier2Backup || opts.EnableTier2Recovery { spec.Tier2 = &kliov1alpha1.Tier2PluginConfiguration{ From 88049747d45d01cc7a5f901dbef7c0b86d180c19 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 15:02:26 +0200 Subject: [PATCH 19/23] test(e2e): count only the cluster's backups in the retention checks The count check already decoded the cluster name of each backup but counted every cluster's backups, while the name listing filtered by cluster. The exact-count assertions of the shared retention flow now look at the cluster under test only. Signed-off-by: Armando Ruocco --- operator/test/klio/features/retention_flow.go | 4 ++-- operator/test/klio/features/tier2_retention.go | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/operator/test/klio/features/retention_flow.go b/operator/test/klio/features/retention_flow.go index 4b4324a8..d2ba8af0 100644 --- a/operator/test/klio/features/retention_flow.go +++ b/operator/test/klio/features/retention_flow.go @@ -91,7 +91,7 @@ func verifyRetentionAndOnDemandApply( // After more backups than the policy keeps, the older ones are deleted. expectedBackups := min(i+1, p.keepLatest) err = wait.For( - checkTierHasBackups(r, p.namespace, p.serverName, p.tierAnnotation, expectedBackups), + checkTierHasBackups(r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation, expectedBackups), wait.WithTimeout(p.retentionTimeout), wait.WithInterval(p.checkInterval), ) @@ -116,7 +116,7 @@ func verifyRetentionAndOnDemandApply( // the one the retention manager deleted (the newest survive). t.Logf("Retention verification: %s should have exactly %d backup(s)", p.tierLabel, p.keepLatest) err := wait.For( - checkTierHasBackups(r, p.namespace, p.serverName, p.tierAnnotation, p.keepLatest), + checkTierHasBackups(r, p.namespace, p.serverName, p.clusterName, p.tierAnnotation, p.keepLatest), wait.WithTimeout(p.retentionTimeout), wait.WithInterval(p.checkInterval), ) diff --git a/operator/test/klio/features/tier2_retention.go b/operator/test/klio/features/tier2_retention.go index 8320a284..97a2224b 100644 --- a/operator/test/klio/features/tier2_retention.go +++ b/operator/test/klio/features/tier2_retention.go @@ -334,6 +334,7 @@ func checkTierHasBackups( r *resources.Resources, namespace string, serverName string, + clusterName string, tierAnnotation string, expectedCount int, ) k8swait.ConditionWithContextFunc { @@ -364,9 +365,13 @@ func checkTierHasBackups( return false, nil //nolint:nilerr } - // Count backups present in the tier (those carrying the tier annotation) + // Count this cluster's backups present in the tier (those carrying the + // tier annotation) count := 0 for i := range backups { + if backups[i].ClusterName != clusterName { + continue + } if backups[i].Annotations[tierAnnotation] == presentAnnotationValue { count++ } From 95838a541902a0b4dd8820af684f6d8f1b1b55e3 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 15:02:26 +0200 Subject: [PATCH 20/23] test(e2e): verify the tier1 WAL horizon follows base retention The tier1 retention feature checked that base backups are deleted but not that the WAL horizon moved with them, which is the part of the WAL cleanup this change touches: the first required WAL is now recomputed from Klio's catalog after retention. After the on-demand apply leaves the newest backup, the test waits until no tier1 WAL older than that backup's begin WAL survives. The tier1 WAL listing helpers move from the e2e package to the Klio features package so both tests can use them. Signed-off-by: Armando Ruocco --- .../web/docs/developer/running-e2e-tests.md | 3 +- operator/test/e2e/maintenance_test.go | 67 ++------------- operator/test/e2e/wal_retention_test.go | 7 +- .../test/klio/features/tier1_retention.go | 48 +++++++++++ operator/test/klio/features/tier1_wal.go | 82 +++++++++++++++++++ 5 files changed, 142 insertions(+), 65 deletions(-) create mode 100644 operator/test/klio/features/tier1_wal.go diff --git a/documentation/web/docs/developer/running-e2e-tests.md b/documentation/web/docs/developer/running-e2e-tests.md index 946db107..494e5b04 100644 --- a/documentation/web/docs/developer/running-e2e-tests.md +++ b/documentation/web/docs/developer/running-e2e-tests.md @@ -132,7 +132,8 @@ The E2E tests are located in `operator/test/e2e/` and include: - **`tier1_retention_test.go`** - Backup retention policy enforcement on a tier1-only deployment: takes more backups than the policy keeps, verifies the retention manager deletes the oldest, then tightens the policy and - verifies `klio retention apply` prunes on demand (`Tier1Retention`) + verifies `klio retention apply` prunes on demand, and that tier1 WALs + older than the surviving backup are removed (`Tier1Retention`) - **`tier2_retention_test.go`** - Backup and WAL retention policy enforcement in tier2 storage: shares the tier1 automatic and on-demand retention flow and adds tier2-only WAL retention and recovery-gate diff --git a/operator/test/e2e/maintenance_test.go b/operator/test/e2e/maintenance_test.go index 50de875a..d030896a 100644 --- a/operator/test/e2e/maintenance_test.go +++ b/operator/test/e2e/maintenance_test.go @@ -20,10 +20,7 @@ SPDX-License-Identifier: Apache-2.0 package e2e import ( - "bytes" "context" - "fmt" - "strings" "testing" "time" @@ -33,65 +30,13 @@ import ( "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/pkg/envconf" + klioFeatures "github.com/cloudnative-pg/klio/operator/test/klio/features" machineryFeatures "github.com/cloudnative-pg/klio/operator/test/machinery/pkg/features" ) // maintenanceClusterName is the CNPG cluster name created by newBackupFeature. const maintenanceClusterName = "test-cluster" -// listTier1WALFiles returns the WAL segment file names stored in tier1 for a -// cluster, sorted ascending. Partial files are excluded. -// -// WAL files live at /klio/data/wal/{clusterName}/{16-char-prefix}/{24-char-name} on -// the server pod; 'find' is unavailable in the minimal container, so we rely on -// shell globbing. -func listTier1WALFiles( - ctx context.Context, - r *resources.Resources, - namespace, podName, clusterName string, -) []string { - var stdout, stderr bytes.Buffer - listCmd := []string{ - "sh", "-c", - fmt.Sprintf("ls /klio/data/wal/%s/*/0000* 2>/dev/null | sort", clusterName), - } - - // ls exits non-zero when nothing matches, which is fine: we return an empty - // list in that case. - _ = r.ExecInPod(ctx, namespace, podName, serverContainerName, listCmd, &stdout, &stderr) - - output := strings.TrimSpace(stdout.String()) - if output == "" { - return nil - } - - var walFiles []string - for line := range strings.SplitSeq(output, "\n") { - parts := strings.Split(line, "/") - name := parts[len(parts)-1] - if strings.HasSuffix(name, ".partial") { - continue - } - walFiles = append(walFiles, name) - } - - return walFiles -} - -// walsOlderThan returns the entries of walFiles that are strictly older than -// boundary. WAL segment names are fixed-width hex, so a lexicographic -// comparison matches WAL ordering. -func walsOlderThan(walFiles []string, boundary string) []string { - var older []string - for _, w := range walFiles { - if w < boundary { - older = append(older, w) - } - } - - return older -} - // assertServerSideMaintenanceRan verifies that, for a tier1-only deployment, // the Klio server applies tier1 WAL retention after a backup completes. // @@ -121,7 +66,7 @@ func assertServerSideMaintenanceRan( boundary := completed.Status.BeginWal require.NotEmpty(t, boundary, "completed backup has no begin WAL in its status") - initial := listTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) + initial := klioFeatures.ListTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) t.Logf("Tier1 WAL files before maintenance settled: %d (%v), begin WAL %q", len(initial), initial, boundary) // Maintenance is asynchronous: the consumer processes the backup after it is @@ -129,17 +74,17 @@ func assertServerSideMaintenanceRan( t.Log("Waiting for server-side tier1 WAL retention to remove WALs older than the backup begin WAL") err = wait.For( func(ctx context.Context) (bool, error) { - walFiles := listTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) - return len(walsOlderThan(walFiles, boundary)) == 0, nil + walFiles := klioFeatures.ListTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) + return len(klioFeatures.WALsOlderThan(walFiles, boundary)) == 0, nil }, wait.WithTimeout(2*time.Minute), wait.WithInterval(10*time.Second), ) - final := listTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) + final := klioFeatures.ListTier1WALFiles(ctx, r, backup.Namespace, serverPodName, maintenanceClusterName) require.NoError(t, err, "server-side tier1 maintenance did not prune WALs older than begin WAL %q; remaining older WALs: %v", - boundary, walsOlderThan(final, boundary)) + boundary, klioFeatures.WALsOlderThan(final, boundary)) // The cluster keeps archiving, and the begin WAL itself is retained, so the // repository must not be empty: an empty result would mean we measured the diff --git a/operator/test/e2e/wal_retention_test.go b/operator/test/e2e/wal_retention_test.go index 79c33cf9..024e00a2 100644 --- a/operator/test/e2e/wal_retention_test.go +++ b/operator/test/e2e/wal_retention_test.go @@ -43,6 +43,7 @@ import ( kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" "github.com/cloudnative-pg/klio/operator/internal/cnpgi" + klioFeatures "github.com/cloudnative-pg/klio/operator/test/klio/features" "github.com/cloudnative-pg/klio/operator/test/klio/infra" "github.com/cloudnative-pg/klio/operator/test/klio/testconfig" machineryConditions "github.com/cloudnative-pg/klio/operator/test/machinery/pkg/conditions" @@ -441,7 +442,7 @@ func (f *WALRetentionFeature) Run() types.StepFunc { walFiles, err := f.scenario.getWALFilesInTier1(ctx, r) require.NoError(t, err, "failed to get WAL files in tier1") t.Logf("Tier1 WAL files before retention advances: %d (%v), boundary %q", len(walFiles), walFiles, boundary) - require.NotEmpty(t, walsOlderThan(walFiles, boundary), + require.NotEmpty(t, klioFeatures.WALsOlderThan(walFiles, boundary), "expected WAL segments older than the second backup begin WAL before retention advances") // Step 4: delete the oldest backup so the retention point can advance to @@ -478,14 +479,14 @@ func (f *WALRetentionFeature) Run() types.StepFunc { func(ctx context.Context) (bool, error) { var err error walFiles, err = f.scenario.getWALFilesInTier1(ctx, r) - return len(walsOlderThan(walFiles, boundary)) == 0, err + return len(klioFeatures.WALsOlderThan(walFiles, boundary)) == 0, err }, wait.WithTimeout(5*time.Minute), wait.WithInterval(10*time.Second), ) require.NoError(t, err, "server-side retention did not prune WALs older than boundary %q; remaining older WALs: %v", - boundary, walsOlderThan(walFiles, boundary)) + boundary, klioFeatures.WALsOlderThan(walFiles, boundary)) require.NotEmpty(t, walFiles, "tier1 WAL repository unexpectedly empty after retention") t.Logf("Server-side WAL retention verified: %d WAL files remain, all >= begin WAL %q", diff --git a/operator/test/klio/features/tier1_retention.go b/operator/test/klio/features/tier1_retention.go index d76466c0..e338010e 100644 --- a/operator/test/klio/features/tier1_retention.go +++ b/operator/test/klio/features/tier1_retention.go @@ -27,6 +27,7 @@ import ( cnpgv1 "github.com/cloudnative-pg/api/pkg/api/v1" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/types" @@ -147,10 +148,57 @@ func (f *Tier1RetentionFeature) Run() types.StepFunc { }, }) + // Base retention moved the WAL horizon: only the newest backup is left, + // so every tier1 WAL older than its begin WAL must be gone too. + verifyTier1WALHorizon(ctx, t, r, f.namespace, f.klioServer.Name, f.clusterName, + f.backups[len(f.backups)-1], f.backupTimeout, f.checkInterval) + return ctx } } +// verifyTier1WALHorizon waits until no tier1 WAL segment older than the begin +// WAL of the given backup survives, which proves the WAL retention horizon was +// recomputed from the catalog after base retention deleted the older backups. +func verifyTier1WALHorizon( + ctx context.Context, + t *testing.T, + r *resources.Resources, + namespace string, + serverName string, + clusterName string, + backup *cnpgv1.Backup, + timeout time.Duration, + interval time.Duration, +) { + t.Helper() + + var newest cnpgv1.Backup + require.NoError(t, r.Get(ctx, backup.Name, namespace, &newest), "failed to refresh backup %s", backup.Name) + boundary := newest.Status.BeginWal + require.NotEmpty(t, boundary, "completed backup %s has no begin WAL in its status", backup.Name) + + podName := serverName + klioPodSuffix + t.Logf("WAL horizon: waiting for tier1 WALs older than %s to be removed", boundary) + err := wait.For( + func(ctx context.Context) (bool, error) { + walFiles := ListTier1WALFiles(ctx, r, namespace, podName, clusterName) + + return len(WALsOlderThan(walFiles, boundary)) == 0, nil + }, + wait.WithTimeout(timeout), + wait.WithInterval(interval), + ) + + final := ListTier1WALFiles(ctx, r, namespace, podName, clusterName) + require.NoError(t, err, "tier1 WALs older than begin WAL %q survived base retention: %v", + boundary, WALsOlderThan(final, boundary)) + // The begin WAL itself is retained, so an empty list means we looked at the + // wrong path rather than at a successful retention. + require.NotEmpty(t, final, "no tier1 WAL files found for cluster %q", clusterName) + t.Logf("PASSED: %d tier1 WAL files remain, all >= %s", len(final), boundary) +} + // Teardown cleans up resources after the test is run. func (f *Tier1RetentionFeature) Teardown() types.StepFunc { return f.teardown diff --git a/operator/test/klio/features/tier1_wal.go b/operator/test/klio/features/tier1_wal.go new file mode 100644 index 00000000..40150e4d --- /dev/null +++ b/operator/test/klio/features/tier1_wal.go @@ -0,0 +1,82 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package features + +import ( + "bytes" + "context" + "fmt" + "strings" + + "sigs.k8s.io/e2e-framework/klient/k8s/resources" +) + +// ListTier1WALFiles returns the WAL segment file names stored in tier1 for a +// cluster, sorted ascending. Partial files are excluded. +// +// WAL files live at /klio/data/wal/{clusterName}/{16-char-prefix}/{24-char-name} on +// the server pod; 'find' is unavailable in the minimal container, so we rely on +// shell globbing. +func ListTier1WALFiles( + ctx context.Context, + r *resources.Resources, + namespace, podName, clusterName string, +) []string { + var stdout, stderr bytes.Buffer + listCmd := []string{ + "sh", "-c", + fmt.Sprintf("ls /klio/data/wal/%s/*/0000* 2>/dev/null | sort", clusterName), + } + + // ls exits non-zero when nothing matches, which is fine: we return an empty + // list in that case. + _ = r.ExecInPod(ctx, namespace, podName, serverContainerName, listCmd, &stdout, &stderr) + + output := strings.TrimSpace(stdout.String()) + if output == "" { + return nil + } + + var walFiles []string + for line := range strings.SplitSeq(output, "\n") { + parts := strings.Split(line, "/") + name := parts[len(parts)-1] + if strings.HasSuffix(name, ".partial") { + continue + } + walFiles = append(walFiles, name) + } + + return walFiles +} + +// WALsOlderThan returns the entries of walFiles that are strictly older than +// boundary. WAL segment names are fixed-width hex, so a lexicographic +// comparison matches WAL ordering. +func WALsOlderThan(walFiles []string, boundary string) []string { + var older []string + for _, w := range walFiles { + if w < boundary { + older = append(older, w) + } + } + + return older +} From 6330c331ef95b90375fd9782ecc854ee95ee7b9f Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 15:11:37 +0200 Subject: [PATCH 21/23] fix(retention): let tier1 delete backups that tier2 retention already removed The tier1 guard kept every backup whose snapshots were missing on tier2, which conflates "not relayed yet" with "relayed and deleted since". With a tier2 policy tighter than tier1's, tier2 retention runs first and removes the older backups; tier1 then kept those same backups, and their WALs, forever, with only an info log that reads as transient. The relay migrates every tier1 snapshot of the cluster at once, so a newer backup complete on tier2 proves the relay ran after the older one existed. A backup absent from tier2 is now deletable when such a newer backup exists; one that is only partially on tier2 is still kept, since a relay may be in flight or have failed midway. Signed-off-by: Armando Ruocco --- core/internal/consumer/maintenance.go | 75 ++++++++++++++++++---- core/internal/consumer/maintenance_test.go | 62 +++++++++++------- 2 files changed, 103 insertions(+), 34 deletions(-) diff --git a/core/internal/consumer/maintenance.go b/core/internal/consumer/maintenance.go index c7e23059..fc9617d5 100644 --- a/core/internal/consumer/maintenance.go +++ b/core/internal/consumer/maintenance.go @@ -96,42 +96,93 @@ func (d *Backup) tier1RetentionGuard( "while listing tier2 snapshots to guard tier1 retention for cluster %q: %w", clusterName, err) } - return keepUntilOnTier2(tier1Snapshots, tier2Snapshots), nil + tier1Backups, err := d.tier1Client.ListBackups(ctx, clusterName) + if err != nil { + return nil, fmt.Errorf( + "while listing tier1 backups to guard tier1 retention for cluster %q: %w", clusterName, err) + } + + return keepUntilOnTier2(tier1Snapshots, tier2Snapshots, tier1Backups), nil } // keepUntilOnTier2 returns a predicate that reports whether a tier1 backup must -// be kept because it is not yet complete on tier2. The relay is a single +// be kept because it has not been relayed to tier2 yet. The relay is a single // snapshot migration with no ordering between a backup's parts, so the tier2 // metadata snapshot alone does not prove the data is there: a backup is // deletable only when every tier1 snapshot of it has a counterpart on tier2. -// A backup the client never meant to relay has nothing to wait for. +// +// A backup with no snapshot at all on tier2 is either not relayed yet or was +// relayed and then deleted by tier2 retention. The relay migrates every tier1 +// snapshot of the cluster at once, so a newer backup complete on tier2 proves +// the relay ran after the older one existed: such a backup is deletable, or +// tier1 would keep it (and its WALs) forever. A backup the client never meant +// to relay has nothing to wait for. func keepUntilOnTier2( tier1Snapshots, tier2Snapshots []kopia.Manifest, + tier1Backups klioclient.BackupList, ) func(backup *klioclient.BackupMetadata) bool { + state := newRelayState(tier1Snapshots, tier2Snapshots) + + var newestComplete int64 + for i := range tier1Backups { + if state.complete(tier1Backups[i].Name) { + newestComplete = max(newestComplete, tier1Backups[i].StartedAt) + } + } + + return func(backup *klioclient.BackupMetadata) bool { + if backup.Annotations[klioclient.Tier2RelayAnnotationName] == klioclient.Tier2RelaySkipped { + return false + } + + // No tier1 snapshot means nothing to protect. + if state.parts[backup.Name] == 0 || state.complete(backup.Name) { + return false + } + + // Partially on tier2: a relay is in flight or failed midway. + if state.relayed[backup.Name] > 0 { + return true + } + + // Absent from tier2: relayed and deleted there only if a newer backup + // went through the relay. + return newestComplete == 0 || backup.StartedAt >= newestComplete + } +} + +// relayState counts, per backup name, the tier1 snapshots and how many of +// them have a counterpart on tier2. +type relayState struct { + parts map[string]int + relayed map[string]int +} + +func newRelayState(tier1Snapshots, tier2Snapshots []kopia.Manifest) relayState { onTier2 := stringset.New() for i := range tier2Snapshots { onTier2.Put(snapshotPartKey(tier2Snapshots[i])) } - incomplete := stringset.New() + state := relayState{parts: make(map[string]int), relayed: make(map[string]int)} for i := range tier1Snapshots { name := tier1Snapshots[i].Tags[klioclient.BackupNameTagName] if name == "" { continue } - if !onTier2.Has(snapshotPartKey(tier1Snapshots[i])) { - incomplete.Put(name) + state.parts[name]++ + if onTier2.Has(snapshotPartKey(tier1Snapshots[i])) { + state.relayed[name]++ } } - return func(backup *klioclient.BackupMetadata) bool { - if backup.Annotations[klioclient.Tier2RelayAnnotationName] == klioclient.Tier2RelaySkipped { - return false - } + return state +} - return incomplete.Has(backup.Name) - } +// complete reports whether every tier1 snapshot of the backup is on tier2. +func (s relayState) complete(name string) bool { + return s.parts[name] > 0 && s.relayed[name] == s.parts[name] } // snapshotPartKey identifies one part of a backup (pgdata, metadata, control diff --git a/core/internal/consumer/maintenance_test.go b/core/internal/consumer/maintenance_test.go index 797f1d84..7cb4a2ab 100644 --- a/core/internal/consumer/maintenance_test.go +++ b/core/internal/consumer/maintenance_test.go @@ -47,9 +47,12 @@ func TestKeepUntilOnTier2(t *testing.T) { snapshot("/tbs/a", "b2", "tablespace", "a"), snapshot("/pgdata", "b3", "pgdata", ""), snapshot("/pgdata_meta", "b3", "metadata", ""), + snapshot("/pgdata", "b0", "pgdata", ""), + snapshot("/pgdata_meta", "b0", "metadata", ""), } // b1 fully relayed, b2 has its metadata but not its tablespace on tier2, - // b3 not relayed at all. + // b3 (newest) not relayed yet, b0 (oldest) absent: relayed with b1 and + // deleted by tier2 retention since. tier2 := []kopia.Manifest{ snapshot("/pgdata", "b1", "pgdata", ""), snapshot("/pgdata_meta", "b1", "metadata", ""), @@ -57,34 +60,49 @@ func TestKeepUntilOnTier2(t *testing.T) { snapshot("/pgdata", "b2", "pgdata", ""), snapshot("/pgdata_meta", "b2", "metadata", ""), } - - keep := keepUntilOnTier2(tier1, tier2) + catalog := klioclient.BackupList{ + {Name: "b0", StartedAt: 50}, + {Name: "b1", StartedAt: 100}, + {Name: "b2", StartedAt: 200}, + {Name: "b3", StartedAt: 300}, + } backup := func(name string) *klioclient.BackupMetadata { + for i := range catalog { + if catalog[i].Name == name { + b := catalog[i] + + return &b + } + } + return &klioclient.BackupMetadata{Name: name} } - if keep(backup("b1")) { - t.Error("kept b1, which is complete on tier2") - } - if !keep(backup("b2")) { - t.Error("did not keep b2, whose tablespace is missing on tier2") - } - if !keep(backup("b3")) { - t.Error("did not keep b3, which is not on tier2") - } - // A backup unknown to tier1 has nothing to protect. - if keep(backup("b4")) { - t.Error("kept b4, which has no tier1 snapshots") - } - // A backup the client never meant to relay is not waited for. skipped := backup("b3") skipped.SetAnnotation(klioclient.Tier2RelayAnnotationName, klioclient.Tier2RelaySkipped) - if keep(skipped) { - t.Error("kept b3, which was never meant to reach tier2") + + tests := []struct { + name string + tier2 []kopia.Manifest + backup *klioclient.BackupMetadata + want bool + }{ + {name: "complete on tier2 is deletable", tier2: tier2, backup: backup("b1"), want: false}, + {name: "tablespace missing on tier2 is kept", tier2: tier2, backup: backup("b2"), want: true}, + {name: "newer than any relayed backup is kept", tier2: tier2, backup: backup("b3"), want: true}, + {name: "older than a relayed backup and gone from tier2 is deletable", tier2: tier2, backup: backup("b0")}, + {name: "unknown to tier1 has nothing to protect", tier2: tier2, backup: backup("b4"), want: false}, + {name: "never meant to reach tier2 is not waited for", tier2: tier2, backup: skipped, want: false}, + {name: "nothing relayed keeps the oldest", tier2: nil, backup: backup("b0"), want: true}, + {name: "nothing relayed keeps the newest", tier2: nil, backup: backup("b3"), want: true}, } - // With no tier2 snapshots, everything on tier1 is kept. - if keepAll := keepUntilOnTier2(tier1, nil); !keepAll(backup("b1")) { - t.Error("keepUntilOnTier2(tier1, nil) did not keep b1") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := keepUntilOnTier2(tier1, tt.tier2, catalog)(tt.backup); got != tt.want { + t.Errorf("keep(%s) = %v, want %v", tt.backup.Name, got, tt.want) + } + }) } } From d937a3f92e2dcb6acacda1eb2802e1e3ba8d09b9 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 15:12:19 +0200 Subject: [PATCH 22/23] fix(operator): make the retention latest field optional in the CRD A required `latest` makes PluginConfigurations stored by earlier versions, which carry the pruned Kopia-style keys and no `latest`, fail schema validation on their next write, including the controller's own status update, unless the cluster ratchets CRD validation. The field is optional with a plain minimum of 1; an empty block converts to "no policy" as before. Signed-off-by: Armando Ruocco --- documentation/web/docs/user/api/_klio_api.md | 6 ++++-- operator/api/v1alpha1/plugin_configuration_types.go | 9 ++++++--- .../crd/bases/klio.cnpg.io_pluginconfigurations.yaml | 12 ++---------- .../dist/chart/crds/pluginconfiguration-crd.yaml | 12 ++---------- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/documentation/web/docs/user/api/_klio_api.md b/documentation/web/docs/user/api/_klio_api.md index c7d491c5..779a23b3 100644 --- a/documentation/web/docs/user/api/_klio_api.md +++ b/documentation/web/docs/user/api/_klio_api.md @@ -206,7 +206,9 @@ _Appears in:_ RetentionPolicy defines which backups Klio should keep. Omitting the whole -policy keeps every backup. +policy, or leaving it empty, keeps every backup. The field is optional so +that objects stored by earlier versions (with the old Kopia-style keys, now +pruned) remain writable after the CRD upgrade. @@ -216,7 +218,7 @@ _Appears in:_ | Field | Description | Required | Default | Validation | | --- | --- | --- | --- | --- | -| `latest` _integer_ | Latest keeps only the given number of most recent backups and deletes the
rest. | True | | | +| `latest` _integer_ | Latest keeps only the given number of most recent backups and deletes the
rest. | | | Minimum: 1
Optional: \{\}
| #### S3Configuration diff --git a/operator/api/v1alpha1/plugin_configuration_types.go b/operator/api/v1alpha1/plugin_configuration_types.go index 92af09d1..b63f60f6 100644 --- a/operator/api/v1alpha1/plugin_configuration_types.go +++ b/operator/api/v1alpha1/plugin_configuration_types.go @@ -206,12 +206,15 @@ func (s *PluginConfigurationSpec) GetWALPrefetch() WALPrefetchConfiguration { } // RetentionPolicy defines which backups Klio should keep. Omitting the whole -// policy keeps every backup. -// +kubebuilder:validation:XValidation:rule="self.latest >= 1",message="latest must be greater than or equal to 1" +// policy, or leaving it empty, keeps every backup. The field is optional so +// that objects stored by earlier versions (with the old Kopia-style keys, now +// pruned) remain writable after the CRD upgrade. type RetentionPolicy struct { // Latest keeps only the given number of most recent backups and deletes the // rest. - Latest int `json:"latest" mapstructure:"latest"` + // +optional + // +kubebuilder:validation:Minimum=1 + Latest int `json:"latest,omitempty" mapstructure:"latest"` } // PluginConfigurationStatus defines the observed state of ClientConfig. diff --git a/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml b/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml index db11bc80..441140c3 100644 --- a/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml +++ b/operator/config/crd/bases/klio.cnpg.io_pluginconfigurations.yaml @@ -1731,13 +1731,9 @@ spec: description: |- Latest keeps only the given number of most recent backups and deletes the rest. + minimum: 1 type: integer - required: - - latest type: object - x-kubernetes-validations: - - message: latest must be greater than or equal to 1 - rule: self.latest >= 1 type: object tier2: description: Tier2 is the Tier 2 configuration @@ -1807,13 +1803,9 @@ spec: description: |- Latest keeps only the given number of most recent backups and deletes the rest. + minimum: 1 type: integer - required: - - latest type: object - x-kubernetes-validations: - - message: latest must be greater than or equal to 1 - rule: self.latest >= 1 type: object x-kubernetes-validations: - message: at least one of enableBackup or enableRecovery must be diff --git a/operator/dist/chart/crds/pluginconfiguration-crd.yaml b/operator/dist/chart/crds/pluginconfiguration-crd.yaml index c07f6dcf..2e225102 100644 --- a/operator/dist/chart/crds/pluginconfiguration-crd.yaml +++ b/operator/dist/chart/crds/pluginconfiguration-crd.yaml @@ -1730,13 +1730,9 @@ spec: description: |- Latest keeps only the given number of most recent backups and deletes the rest. + minimum: 1 type: integer - required: - - latest type: object - x-kubernetes-validations: - - message: latest must be greater than or equal to 1 - rule: self.latest >= 1 type: object tier2: description: Tier2 is the Tier 2 configuration @@ -1806,13 +1802,9 @@ spec: description: |- Latest keeps only the given number of most recent backups and deletes the rest. + minimum: 1 type: integer - required: - - latest type: object - x-kubernetes-validations: - - message: latest must be greater than or equal to 1 - rule: self.latest >= 1 type: object x-kubernetes-validations: - message: at least one of enableBackup or enableRecovery must be From 82f05d1f04c7d9e3b7aed6a6a941be49b1b20ae7 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 14 Sep 2026 15:12:19 +0200 Subject: [PATCH 23/23] docs(agents): list the retention config files among the synced pairs Signed-off-by: Armando Ruocco --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 6aaa0494..c05ccfd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,6 +139,7 @@ Klio-only assertions) must live outside `machinery` — e.g. under - `operator/pkg/config/server.go` ↔ `core/pkg/config/server.go` - `operator/pkg/config/client.go` ↔ `core/pkg/config/client.go` - `operator/pkg/config/compression.go` ↔ `core/pkg/config/compression.go` + - `operator/pkg/config/retention.go` ↔ `core/pkg/config/retention.go` - When you change a metric in `core/internal/opentelemetry/catalog.go` (rename, add, remove, or change a metric's unit, type, or attributes),