From 8f55350f8b37b30d0a66a350e11981fc560a0d0d Mon Sep 17 00:00:00 2001 From: Bram Schuur Date: Thu, 16 Jul 2026 10:59:57 +0200 Subject: [PATCH 1/3] STAC-24630: Add stackgraph-v2 backup/restore --- cmd/root.go | 5 + cmd/settings/restore.go | 2 +- cmd/stackgraph/restore.go | 2 +- cmd/stackgraphv2/abort.go | 67 +++ cmd/stackgraphv2/backfill.go | 74 ++++ cmd/stackgraphv2/check_and_finalize.go | 58 +++ cmd/stackgraphv2/list.go | 102 +++++ cmd/stackgraphv2/restore.go | 415 ++++++++++++++++++ cmd/stackgraphv2/restore_test.go | 125 ++++++ cmd/stackgraphv2/simplejob.go | 134 ++++++ cmd/stackgraphv2/stackgraphv2.go | 21 + cmd/victoriametrics/restore.go | 2 +- internal/clients/s3/filter.go | 17 + internal/orchestration/restore/job.go | 3 + .../scripts/restore-settings-backup.sh | 2 +- .../restore-stackgraph-backup-v2-abort.sh | 13 + .../restore-stackgraph-backup-v2-backfill.sh | 13 + .../restore-stackgraph-backup-v2-env.sh | 30 ++ .../restore-stackgraph-backup-v2-live.sh | 38 ++ .../scripts/restore-stackgraph-backup.sh | 6 +- .../restore-victoria-metrics-backup.sh | 2 +- 21 files changed, 1123 insertions(+), 8 deletions(-) create mode 100644 cmd/stackgraphv2/abort.go create mode 100644 cmd/stackgraphv2/backfill.go create mode 100644 cmd/stackgraphv2/check_and_finalize.go create mode 100644 cmd/stackgraphv2/list.go create mode 100644 cmd/stackgraphv2/restore.go create mode 100644 cmd/stackgraphv2/restore_test.go create mode 100644 cmd/stackgraphv2/simplejob.go create mode 100644 cmd/stackgraphv2/stackgraphv2.go create mode 100644 internal/scripts/scripts/restore-stackgraph-backup-v2-abort.sh create mode 100644 internal/scripts/scripts/restore-stackgraph-backup-v2-backfill.sh create mode 100644 internal/scripts/scripts/restore-stackgraph-backup-v2-env.sh create mode 100644 internal/scripts/scripts/restore-stackgraph-backup-v2-live.sh diff --git a/cmd/root.go b/cmd/root.go index f2d7686..cba0f17 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/stackvista/stackstate-backup-cli/cmd/elasticsearch" "github.com/stackvista/stackstate-backup-cli/cmd/settings" "github.com/stackvista/stackstate-backup-cli/cmd/stackgraph" + "github.com/stackvista/stackstate-backup-cli/cmd/stackgraphv2" "github.com/stackvista/stackstate-backup-cli/cmd/version" "github.com/stackvista/stackstate-backup-cli/cmd/victoriametrics" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" @@ -42,6 +43,10 @@ func init() { addBackupConfigFlags(stackgraphCmd) rootCmd.AddCommand(stackgraphCmd) + stackgraphv2Cmd := stackgraphv2.Cmd(flags) + addBackupConfigFlags(stackgraphv2Cmd) + rootCmd.AddCommand(stackgraphv2Cmd) + settingsCmd := settings.Cmd(flags) addBackupConfigFlags(settingsCmd) rootCmd.AddCommand(settingsCmd) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 05e09c8..86742b5 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -197,7 +197,7 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En commonVar := []corev1.EnvVar{ {Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket}, {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, - {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 8391ceb..4c02c06 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -276,7 +276,7 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, - {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, diff --git a/cmd/stackgraphv2/abort.go b/cmd/stackgraphv2/abort.go new file mode 100644 index 0000000..24fa11f --- /dev/null +++ b/cmd/stackgraphv2/abort.go @@ -0,0 +1,67 @@ +package stackgraphv2 + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" + "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" +) + +const ( + abortNameTemplate = "stackgraph-abort-v2" + abortScript = "/backup-restore-scripts/restore-stackgraph-backup-v2-abort.sh" +) + +func abortCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "abort", + Short: "Abort a partially backfilled restore.", + Long: "Abort a partially backfilled restore. This will not load (backfill) any data but keep the data as is." + + "Be aware this leaves the restore incomplete, historical data will not be recovered, but leave the instance in a usable state." + + "This should be used when backfilling is failing and the database restore is accepted as-is.", + Run: func(_ *cobra.Command, _ []string) { + cmdutils.Run(globalFlags, runAbort, cmdutils.StorageIsRequired) + }, + } + + return cmd +} + +func runAbort(appCtx *app.Context) error { + // Setup Kubernetes resources for restore job + appCtx.Logger.Println() + if err := restore.EnsureResources(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger); err != nil { + return err + } + + // Create restore job + appCtx.Logger.Println() + appCtx.Logger.Infof("Creating abort job.") + + jobName := fmt.Sprintf("%s-%s", abortNameTemplate, time.Now().Format("20060102t150405")) + + if err := createAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Config); err != nil { + return fmt.Errorf("failed to create abort job: %w", err) + } + + appCtx.Logger.Successf("Abort job created: %s", jobName) + + return waitAndCleanupAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) +} + +// waitAndCleanupAbortJob waits for job completion and cleans up resources +func waitAndCleanupAbortJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error { + restore.PrintWaitingMessage(log, "stackgraph-v2", jobName, namespace) + return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, false) +} + +// createAbortJob creates a Kubernetes Job for aborting a partially backfilled restore +func createAbortJob(k8sClient *k8s.Client, namespace, jobName string, config *config.Config) error { + return createSimpleJob(k8sClient, namespace, jobName, "abort", abortScript, config) +} diff --git a/cmd/stackgraphv2/backfill.go b/cmd/stackgraphv2/backfill.go new file mode 100644 index 0000000..f0a8ff9 --- /dev/null +++ b/cmd/stackgraphv2/backfill.go @@ -0,0 +1,74 @@ +package stackgraphv2 + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" + "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" +) + +const ( + backfillNameTemplate = "stackgraph-backfill-v2" + backfillScript = "/backup-restore-scripts/restore-stackgraph-backup-v2-backfill.sh" +) + +func backfillCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "backfill", + Short: "Complete a restore by backfilling old data", + Long: "Complete a restore by backfilling old data. This process can run while the system is already up and running " + + "After the backfill is done, the restore process is complete. ", + Run: func(_ *cobra.Command, _ []string) { + cmdutils.Run(globalFlags, runBackfill, cmdutils.StorageIsRequired) + }, + } + + return cmd +} + +func runBackfill(appCtx *app.Context) error { + // Setup Kubernetes resources for restore job + appCtx.Logger.Println() + if err := restore.EnsureResources(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger); err != nil { + return err + } + + // Create restore job + appCtx.Logger.Println() + appCtx.Logger.Infof("Creating backfill job.") + + jobName := fmt.Sprintf("%s-%s", backfillNameTemplate, time.Now().Format("20060102t150405")) + + if err := createBackfillJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Config); err != nil { + return fmt.Errorf("failed to create backfill job: %w", err) + } + + appCtx.Logger.Successf("Backfill job created: %s", jobName) + + err := waitAndCleanupBackfillJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) + + if err != nil { + appCtx.Logger.Println() + appCtx.Logger.Infof("Backfill failed. It is possible to restart the backfill without starting a complete restore.") + return err + } + + return nil +} + +// waitAndCleanupBackfillJob waits for job completion and cleans up resources +func waitAndCleanupBackfillJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error { + restore.PrintWaitingMessage(log, "stackgraph-v2", jobName, namespace) + return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, false) +} + +// createBackfillJob creates a Kubernetes Job for backfilling old data during a restore +func createBackfillJob(k8sClient *k8s.Client, namespace, jobName string, config *config.Config) error { + return createSimpleJob(k8sClient, namespace, jobName, "backfill", backfillScript, config) +} diff --git a/cmd/stackgraphv2/check_and_finalize.go b/cmd/stackgraphv2/check_and_finalize.go new file mode 100644 index 0000000..39cd794 --- /dev/null +++ b/cmd/stackgraphv2/check_and_finalize.go @@ -0,0 +1,58 @@ +package stackgraphv2 + +import ( + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" + "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale" +) + +// Check and finalize command flags +var ( + checkJobName string + waitForJob bool +) + +func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "check-and-finalize", + Short: "Check and finalize a Stackgraph restore (v2) job", + Long: `Check the status of a background Stackgraph restore job and clean up resources. + +This command is useful when a restore job was started with --background flag or was interrupted (Ctrl+C). +It will check the job status, print logs if it failed, and clean up the job and PVC resources. + +Examples: + # Check job status without waiting + sts-backup stackgraph-v2 check-and-finalize --job stackgraph-restore-20250128t143000 -n my-namespace + + # Wait for job completion and cleanup + sts-backup stackgraph-v2 check-and-finalize --job stackgraph-restore-20250128t143000 --wait -n my-namespace`, + Run: func(_ *cobra.Command, _ []string) { + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired) + }, + } + + cmd.Flags().StringVarP(&checkJobName, "job", "j", "", "Stackgraph restore job name (required)") + cmd.Flags().BoolVarP(&waitForJob, "wait", "w", false, "Wait for job to complete before cleanup") + _ = cmd.MarkFlagRequired("job") + + return cmd +} + +func runCheckAndFinalize(appCtx *app.Context) error { + return restore.CheckAndFinalize(restore.CheckAndFinalizeParams{ + K8sClient: appCtx.K8sClient, + Namespace: appCtx.Namespace, + JobName: checkJobName, + ServiceName: "stackgraph", + ScaleUpFn: scale.ScaleUpAndReleaseLock, + ScaleDownFn: scale.ScaleDown, + ScaleSelector: appCtx.Config.Stackgraph.Restore.ScaleDownLabelSelector, + CleanupPVC: true, + WaitForJob: waitForJob, + Log: appCtx.Logger, + }) +} diff --git a/cmd/stackgraphv2/list.go b/cmd/stackgraphv2/list.go new file mode 100644 index 0000000..7700379 --- /dev/null +++ b/cmd/stackgraphv2/list.go @@ -0,0 +1,102 @@ +package stackgraphv2 + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" + "github.com/stackvista/stackstate-backup-cli/internal/app" + s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/output" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" +) + +const ( + backupFileNameRegex = `^sts-backup-.*\.graph.v2$` +) + +func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List available Stackgraph backups (v2) from S3", + Run: func(_ *cobra.Command, _ []string) { + cmdutils.Run(globalFlags, runList, cmdutils.StorageIsRequired) + }, + } +} + +func runList(appCtx *app.Context) error { + // Setup port-forward to S3-compatible storage + storageService := appCtx.Config.GetStorageService() + serviceName := storageService.Name + remotePort := storageService.Port + + pf, err := portforward.SetupPortForward(appCtx.K8sClient, appCtx.Namespace, serviceName, remotePort, appCtx.Logger) + if err != nil { + return err + } + defer close(pf.StopChan) + + // Create S3 client with actual port + s3Client, err := appCtx.NewS3Client(pf.LocalPort) + if err != nil { + return fmt.Errorf("failed to create S3 client: %w", err) + } + + // List objects in bucket + bucket := appCtx.Config.Stackgraph.Bucket + prefix := appCtx.Config.Stackgraph.S3Prefix + "v2/" + + appCtx.Logger.Infof("Listing Stackgraph backups in bucket '%s' with prefix '%s'...", bucket, prefix) + + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + } + + result, err := s3Client.ListObjectsV2(context.Background(), input) + if err != nil { + return fmt.Errorf("failed to list S3 objects: %w", err) + } + + filteredObjects := s3client.ConvertBackupObjects(result.Contents) + + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return fmt.Errorf("failed to filter objects: %w", err) + } + + // Sort by LastModified time (most recent first) + sort.Slice(filteredObjects, func(i, j int) bool { + return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) + }) + + if len(filteredObjects) == 0 { + appCtx.Formatter.PrintMessage("No backups found") + return nil + } + + table := output.Table{ + Headers: []string{"NAME", "LAST MODIFIED"}, + Rows: make([][]string, 0, len(filteredObjects)), + } + + for _, obj := range filteredObjects { + row := []string{ + strings.TrimPrefix(obj.Key, prefix), + obj.LastModified.Format("2006-01-02 15:04:05 MST"), + } + table.Rows = append(table.Rows, row) + } + + return appCtx.Formatter.PrintTable(table) +} diff --git a/cmd/stackgraphv2/restore.go b/cmd/stackgraphv2/restore.go new file mode 100644 index 0000000..c42ec8d --- /dev/null +++ b/cmd/stackgraphv2/restore.go @@ -0,0 +1,415 @@ +package stackgraphv2 + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" + "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" + s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale" + corev1 "k8s.io/api/core/v1" +) + +const ( + jobNameTemplate = "stackgraph-restore-v2" + configMapDefaultFileMode = 0755 + purgeStackgraphDataFlag = "-force" +) + +// Restore command flags +var ( + archiveName string + useLatest bool + skipConfirmation bool + skipStackpacks bool +) + +func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "restore", + Short: "Restore Stackgraph from a backup archive", + Long: "Restore Stackgraph data from a backup archive stored in S3. Automatically also restores " + + "Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. " + + "Can use --latest or --archive to specify which backup to restore." + + "The process is split into two phases: restoring live data, which takes the system down. And a backfill stage which loads historic data.", + Run: func(_ *cobra.Command, _ []string) { + cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired) + }, + } + + cmd.Flags().StringVar(&archiveName, "archive", "", "Specific archive name to restore (e.g., sts-backup-20210216-0300.graph)") + cmd.Flags().BoolVar(&useLatest, "latest", false, "Restore from the most recent backup") + cmd.Flags().BoolVarP(&skipConfirmation, "yes", "y", false, "Skip confirmation prompt") + cmd.Flags().BoolVar(&skipStackpacks, "skip-stackpacks", false, "Skip restoring stackpacks backup") + cmd.MarkFlagsMutuallyExclusive("archive", "latest") + cmd.MarkFlagsOneRequired("archive", "latest") + + return cmd +} + +func runRestore(appCtx *app.Context) error { + err := liveRestore(appCtx) + if err != nil { + return err + } + + appCtx.Logger.Println() + appCtx.Logger.Infof("The system is back up and accessible. Continuing with backfilling historical data, this will happen while the system is running.") + appCtx.Logger.Infof("During this process not all historical data might be available.") + + return runBackfill(appCtx) +} + +func liveRestore(appCtx *app.Context) error { + // Determine which archive to restore + backupFile := archiveName + if useLatest { + appCtx.Logger.Infof("Finding latest backup...") + latest, err := getLatestBackup(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger) + if err != nil { + return err + } + backupFile = latest + appCtx.Logger.Infof("Using latest backup: %s", backupFile) + } + + // Warn user and ask for confirmation + if !skipConfirmation { + appCtx.Logger.Println() + appCtx.Logger.Warningf("WARNING: Restoring from backup will PURGE all existing Stackgraph data!") + appCtx.Logger.Warningf("This operation cannot be undone.") + appCtx.Logger.Println() + appCtx.Logger.Infof("Backup to restore: %s", backupFile) + appCtx.Logger.Infof("Namespace: %s", appCtx.Namespace) + appCtx.Logger.Println() + + if !restore.PromptForConfirmation() { + return fmt.Errorf("restore operation cancelled by user") + } + } + + // Scale down deployments before restore (with lock protection) + appCtx.Logger.Println() + scaleDownLabelSelector := appCtx.Config.Stackgraph.Restore.ScaleDownLabelSelector + scaledDeployments, err := scale.ScaleDownWithLock(scale.ScaleDownWithLockParams{ + K8sClient: appCtx.K8sClient, + Namespace: appCtx.Namespace, + LabelSelector: scaleDownLabelSelector, + Datastore: config.DatastoreStackgraph, + AllSelectors: appCtx.Config.GetAllScaleDownSelectors(), + Log: appCtx.Logger, + }) + if err != nil { + return err + } + + // Ensure deployments are scaled back up and lock released on exit (even if restore fails) + defer func() { + if len(scaledDeployments) > 0 { + appCtx.Logger.Println() + if err := scale.ScaleUpAndReleaseLock(appCtx.K8sClient, appCtx.Namespace, scaleDownLabelSelector, appCtx.Logger); err != nil { + appCtx.Logger.Warningf("Failed to scale up deployments: %v", err) + } + } + }() + + // Setup Kubernetes resources for restore job + appCtx.Logger.Println() + if err := restore.EnsureResources(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger); err != nil { + return err + } + + // Create restore job + appCtx.Logger.Println() + appCtx.Logger.Infof("Creating restore of live data job for backup: %s", backupFile) + + jobName := fmt.Sprintf("%s-%s", jobNameTemplate, time.Now().Format("20060102t150405")) + + if err = createRestoreJob(appCtx.K8sClient, appCtx.Namespace, jobName, backupFile, appCtx.Config); err != nil { + return fmt.Errorf("failed to create restore job: %w", err) + } + + appCtx.Logger.Successf("Restore job created: %s", jobName) + + err = waitAndCleanupRestoreJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) + if err != nil { + return err + } + + appCtx.Logger.Println() + appCtx.Logger.Infof("Successfully restored live data for: %s", backupFile) + return nil +} + +// waitAndCleanupRestoreJob waits for job completion and cleans up resources +func waitAndCleanupRestoreJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error { + restore.PrintWaitingMessage(log, "stackgraph-v2", jobName, namespace) + return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, true) +} + +// getLatestBackup retrieves the most recent backup from S3 +func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Config, log *logger.Logger) (string, error) { + // Setup port-forward to S3-compatible storage + storageService := config.GetStorageService() + serviceName := storageService.Name + remotePort := storageService.Port + + pf, err := portforward.SetupPortForward(k8sClient, namespace, serviceName, remotePort, log) + if err != nil { + return "", err + } + defer close(pf.StopChan) + + // Create S3 client + endpoint := fmt.Sprintf("http://localhost:%d", pf.LocalPort) + s3Client, err := s3client.NewClient(endpoint, config.GetStorageAccessKey(), config.GetStorageSecretKey()) + if err != nil { + return "", err + } + + // List objects in bucket + bucket := config.Stackgraph.Bucket + prefix := config.Stackgraph.S3Prefix + "v2/" + + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + } + + result, err := s3Client.ListObjectsV2(context.Background(), input) + if err != nil { + return "", fmt.Errorf("failed to list S3 objects: %w", err) + } + + filteredObjects := s3client.ConvertBackupObjects(result.Contents) + + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return "", fmt.Errorf("failed to filter objects: %w", err) + } + + if len(filteredObjects) == 0 { + return "", fmt.Errorf("no backups found in bucket %s", bucket) + } + + // Sort by LastModified time (most recent first) + sort.Slice(filteredObjects, func(i, j int) bool { + return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) + }) + return filteredObjects[0].Key, nil +} + +// buildPVCSpec builds a PVCSpec from configuration +func buildPVCSpec(name string, config *config.Config, labels map[string]string) k8s.PVCSpec { + pvcConfig := config.Stackgraph.Restore.PVC + + // Convert string access modes to k8s types + accessModes := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} // default + if len(pvcConfig.AccessModes) > 0 { + accessModes = make([]corev1.PersistentVolumeAccessMode, 0, len(pvcConfig.AccessModes)) + for _, mode := range pvcConfig.AccessModes { + accessModes = append(accessModes, corev1.PersistentVolumeAccessMode(mode)) + } + } + + // Handle storage class (nil if not set) + var storageClass *string + if pvcConfig.StorageClassName != "" { + storageClass = &pvcConfig.StorageClassName + } + + return k8s.PVCSpec{ + Name: name, + Labels: labels, + // This is not derived from the restore pvc size, the tmp pvc for restore only needs a bounded buffer for stackpacks. + StorageSize: "5Gi", + AccessModes: accessModes, + StorageClass: storageClass, + } +} + +// createRestoreJob creates a Kubernetes Job and PVC for restoring from backup +func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile string, config *config.Config) error { + defaultMode := int32(configMapDefaultFileMode) + + // Merge common labels with resource-specific labels + pvcLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, map[string]string{}) + jobLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, config.Stackgraph.Restore.Job.Labels) + + // Create PVC first + pvcSpec := buildPVCSpec(jobName, config, pvcLabels) + pvc, err := k8sClient.CreatePVC(namespace, pvcSpec) + if err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + + // Build job spec using configuration + spec := k8s.JobSpec{ + Name: jobName, + Labels: jobLabels, + ImagePullSecrets: k8s.ConvertImagePullSecrets(config.Stackgraph.Restore.Job.ImagePullSecrets), + SecurityContext: k8s.ConvertPodSecurityContext(&config.Stackgraph.Restore.Job.SecurityContext), + NodeSelector: config.Stackgraph.Restore.Job.NodeSelector, + Tolerations: k8s.ConvertTolerations(config.Stackgraph.Restore.Job.Tolerations), + Affinity: k8s.ConvertAffinity(config.Stackgraph.Restore.Job.Affinity), + Containers: buildRestoreContainers(backupFile, config), + InitContainers: buildRestoreInitContainers(config), + Volumes: buildRestoreVolumes(jobName, config, defaultMode), + } + + // Create job + _, err = k8sClient.CreateJob(namespace, spec) + if err != nil { + // Cleanup PVC if job creation fails + _ = k8sClient.DeletePVC(namespace, pvc.Name) + return fmt.Errorf("failed to create job: %w", err) + } + + return nil +} + +// buildRestoreEnvVars constructs environment variables for the restore job +func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvVar { + storageService := config.GetStorageService() + env := []corev1.EnvVar{ + {Name: "BACKUP_FILE", Value: backupFile}, + {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, + {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, + {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, + {Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, + {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, + {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, + {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks || config.Stackpacks == nil)}, + } + if config.Stackpacks != nil { + env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksURI}) + env = append(env, corev1.EnvVar{Name: "BACKUP_STACKGRAPH_STACKPACKS_DIR", Value: config.Stackpacks.BackupDirectory}) + } + return env +} + +// buildRestoreVolumeMounts constructs volume mounts for the restore job container +func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, + {Name: "minio-keys", MountPath: "/aws-keys"}, + {Name: "tmp-data", MountPath: "/tmp-data"}, + } + + if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksURI, "file://") { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "stackpacks-local", + MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksURI, "file://"), + }) + } + + return volumeMounts +} + +// buildRestoreInitContainers constructs init containers for the restore job +func buildRestoreInitContainers(config *config.Config) []corev1.Container { + storageService := config.GetStorageService() + return []corev1.Container{ + { + Name: "wait", + Image: config.Stackgraph.Restore.Job.WaitImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sh", + "-c", + fmt.Sprintf("/entrypoint -c %s:%d -t 300", storageService.Name, storageService.Port), + }, + SecurityContext: k8s.ConvertSecurityContext(config.Stackgraph.Restore.Job.ContainerSecurityContext), + }, + } +} + +// buildRestoreVolumes constructs volumes for the restore job pod +func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int32) []corev1.Volume { + volumes := []corev1.Volume{ + { + Name: "backup-log", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Stackgraph.Restore.LoggingConfigConfigMapName, + }, + }, + }, + }, + { + Name: "backup-restore-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: restore.RestoreScriptsConfigMap, + }, + DefaultMode: &defaultMode, + }, + }, + }, + { + Name: "minio-keys", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: restore.MinioKeysSecretName, + }, + }, + }, + { + Name: "tmp-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: jobName, + }, + }, + }, + } + if config.Stackpacks != nil && config.Stackpacks.PVC != "" { + volumes = append(volumes, corev1.Volume{ + Name: "stackpacks-local", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Stackpacks.PVC, + }, + }, + }) + } + + return volumes +} + +// buildRestoreContainers constructs containers for the restore job +func buildRestoreContainers(backupFile string, config *config.Config) []corev1.Container { + return []corev1.Container{ + { + Name: "restore", + Image: config.Stackgraph.Restore.Job.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: k8s.ConvertSecurityContext(config.Stackgraph.Restore.Job.ContainerSecurityContext), + Command: []string{"/backup-restore-scripts/restore-stackgraph-backup-v2-live.sh"}, + Env: buildRestoreEnvVars(backupFile, config), + Resources: k8s.ConvertResources(config.Stackgraph.Restore.Job.Resources), + VolumeMounts: buildRestoreVolumeMounts(config), + }, + } +} diff --git a/cmd/stackgraphv2/restore_test.go b/cmd/stackgraphv2/restore_test.go new file mode 100644 index 0000000..42a1939 --- /dev/null +++ b/cmd/stackgraphv2/restore_test.go @@ -0,0 +1,125 @@ +package stackgraphv2 + +import ( + "testing" + + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stretchr/testify/assert" +) + +func TestBuildRestoreEnvVars_SkipStackpacksWhenMissing(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + skipStackpacksFlag bool + expectedValue string + }{ + { + name: "stackpacks nil and flag false", + stackpacks: nil, + skipStackpacksFlag: false, + expectedValue: "true", + }, + { + name: "stackpacks nil and flag true", + stackpacks: nil, + skipStackpacksFlag: true, + expectedValue: "true", + }, + { + name: "stackpacks present and flag false", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: false, + expectedValue: "false", + }, + { + name: "stackpacks present and flag true", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: true, + expectedValue: "true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set the package-level flag + skipStackpacks = tt.skipStackpacksFlag + + cfg := &config.Config{ + Stackpacks: tt.stackpacks, + Storage: config.StorageConfig{ + Service: config.ServiceConfig{Name: "storage", Port: 9000}, + }, + } + envVars := buildRestoreEnvVars("backup.graph", cfg) + + var skipValue string + for _, env := range envVars { + if env.Name == "SKIP_STACKPACKS" { + skipValue = env.Value + break + } + } + assert.Equal(t, tt.expectedValue, skipValue) + }) + } +} + +func TestBuildRestoreVolumeMounts_StackpacksLocalFileURI(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + expectStackpacks bool + expectedMountPath string + }{ + { + name: "no stackpacks config", + stackpacks: nil, + expectStackpacks: false, + }, + { + name: "stackpacks with no PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local"}, + expectStackpacks: false, + }, + { + name: "stackpacks with file:// URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, + expectStackpacks: true, + expectedMountPath: "/var/stackpacks_local", + }, + { + name: "stackpacks with non-file URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, + expectStackpacks: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &config.Config{Stackpacks: tt.stackpacks} + mounts := buildRestoreVolumeMounts(cfg) + + var stackpacksMount *struct{ Name, MountPath string } + for _, m := range mounts { + if m.Name == "stackpacks-local" { + stackpacksMount = &struct{ Name, MountPath string }{m.Name, m.MountPath} + break + } + } + + if tt.expectStackpacks { + assert.NotNil(t, stackpacksMount, "expected stackpacks-local volume mount to be present") + assert.Equal(t, tt.expectedMountPath, stackpacksMount.MountPath) + } else { + assert.Nil(t, stackpacksMount, "expected stackpacks-local volume mount to be absent") + } + }) + } +} diff --git a/cmd/stackgraphv2/simplejob.go b/cmd/stackgraphv2/simplejob.go new file mode 100644 index 0000000..e6ca0e2 --- /dev/null +++ b/cmd/stackgraphv2/simplejob.go @@ -0,0 +1,134 @@ +package stackgraphv2 + +import ( + "fmt" + + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" + corev1 "k8s.io/api/core/v1" +) + +// createSimpleJob creates a Kubernetes Job for a "simple" stackgraph restore step +// (abort or backfill). These jobs share the same spec and only differ in their +// container name and command. +func createSimpleJob(k8sClient *k8s.Client, namespace, jobName, containerName, command string, config *config.Config) error { + defaultMode := int32(configMapDefaultFileMode) + + // Merge common labels with resource-specific labels + jobLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, config.Stackgraph.Restore.Job.Labels) + + // Build job spec using configuration + spec := k8s.JobSpec{ + Name: jobName, + Labels: jobLabels, + ImagePullSecrets: k8s.ConvertImagePullSecrets(config.Stackgraph.Restore.Job.ImagePullSecrets), + SecurityContext: k8s.ConvertPodSecurityContext(&config.Stackgraph.Restore.Job.SecurityContext), + NodeSelector: config.Stackgraph.Restore.Job.NodeSelector, + Tolerations: k8s.ConvertTolerations(config.Stackgraph.Restore.Job.Tolerations), + Affinity: k8s.ConvertAffinity(config.Stackgraph.Restore.Job.Affinity), + Containers: buildSimpleJobContainers(config, containerName, command), + InitContainers: buildSimpleJobInitContainers(config), + Volumes: buildSimpleJobVolumes(config, defaultMode), + } + + // Create job + _, err := k8sClient.CreateJob(namespace, spec) + if err != nil { + return fmt.Errorf("failed to create job: %w", err) + } + + return nil +} + +// buildSimpleJobEnvVars constructs environment variables shared by the abort and backfill jobs +func buildSimpleJobEnvVars(config *config.Config) []corev1.EnvVar { + storageService := config.GetStorageService() + return []corev1.EnvVar{ + {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, + {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, + {Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, + {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, + {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, + {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, + } +} + +// buildSimpleJobVolumeMounts constructs volume mounts shared by the abort and backfill job containers +func buildSimpleJobVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, + {Name: "minio-keys", MountPath: "/aws-keys"}, + } +} + +// buildSimpleJobInitContainers constructs init containers shared by the abort and backfill jobs +func buildSimpleJobInitContainers(config *config.Config) []corev1.Container { + storageService := config.GetStorageService() + return []corev1.Container{ + { + Name: "wait", + Image: config.Stackgraph.Restore.Job.WaitImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sh", + "-c", + fmt.Sprintf("/entrypoint -c %s:%d -t 300", storageService.Name, storageService.Port), + }, + SecurityContext: k8s.ConvertSecurityContext(config.Stackgraph.Restore.Job.ContainerSecurityContext), + }, + } +} + +// buildSimpleJobVolumes constructs volumes shared by the abort and backfill job pods +func buildSimpleJobVolumes(config *config.Config, defaultMode int32) []corev1.Volume { + return []corev1.Volume{ + { + Name: "backup-log", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Stackgraph.Restore.LoggingConfigConfigMapName, + }, + }, + }, + }, + { + Name: "backup-restore-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: restore.RestoreScriptsConfigMap, + }, + DefaultMode: &defaultMode, + }, + }, + }, + { + Name: "minio-keys", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: restore.MinioKeysSecretName, + }, + }, + }, + } +} + +// buildSimpleJobContainers constructs the main container for a simple (abort/backfill) job +func buildSimpleJobContainers(config *config.Config, name, command string) []corev1.Container { + return []corev1.Container{ + { + Name: name, + Image: config.Stackgraph.Restore.Job.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: k8s.ConvertSecurityContext(config.Stackgraph.Restore.Job.ContainerSecurityContext), + Command: []string{command}, + Env: buildSimpleJobEnvVars(config), + Resources: k8s.ConvertResources(config.Stackgraph.Restore.Job.Resources), + VolumeMounts: buildSimpleJobVolumeMounts(), + }, + } +} diff --git a/cmd/stackgraphv2/stackgraphv2.go b/cmd/stackgraphv2/stackgraphv2.go new file mode 100644 index 0000000..1779a10 --- /dev/null +++ b/cmd/stackgraphv2/stackgraphv2.go @@ -0,0 +1,21 @@ +package stackgraphv2 + +import ( + "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" +) + +func Cmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "stackgraph-v2", + Short: "Stackgraph backup and restore (v2) operations", + } + + cmd.AddCommand(listCmd(globalFlags)) + cmd.AddCommand(restoreCmd(globalFlags)) + cmd.AddCommand(backfillCmd(globalFlags)) + cmd.AddCommand(abortCmd(globalFlags)) + cmd.AddCommand(checkAndFinalizeCmd(globalFlags)) + + return cmd +} diff --git a/cmd/victoriametrics/restore.go b/cmd/victoriametrics/restore.go index 6c27f2d..42e2471 100644 --- a/cmd/victoriametrics/restore.go +++ b/cmd/victoriametrics/restore.go @@ -230,7 +230,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri func buildRestoreEnvVars(config *config.Config) []corev1.EnvVar { storageService := config.GetStorageService() return []corev1.EnvVar{ - {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, } } diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index 34e9e1e..839628b 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -38,6 +38,23 @@ func FilterBackupObjects(objects []s3types.Object) []Object { return filteredObjects } +// ConvertBackupObjects filters out backup part files ending with .digits. +func ConvertBackupObjects(objects []s3types.Object) []Object { + var filteredObjects []Object + + for _, obj := range objects { + key := aws.ToString(obj.Key) + + filteredObjects = append(filteredObjects, Object{ + Key: key, + LastModified: aws.ToTime(obj.LastModified), + Size: aws.ToInt64(obj.Size), + }) + } + + return filteredObjects +} + func hasNumericFileSuffix(key string) bool { if !strings.Contains(key, ".") { return false diff --git a/internal/orchestration/restore/job.go b/internal/orchestration/restore/job.go index dab9971..d5925cf 100644 --- a/internal/orchestration/restore/job.go +++ b/internal/orchestration/restore/job.go @@ -84,6 +84,9 @@ func PrintWaitingMessage(log *logger.Logger, serviceName, jobName, namespace str log.Println() log.Infof("Waiting for restore job to complete (this may take significant amount of time depending on the archive size)...") log.Println() + log.Infof("Monitoring commands:") + log.Infof(" kubectl logs --follow job/%s -n %s", jobName, namespace) + log.Println() log.Infof("You can safely interrupt this command with Ctrl+C.") log.Infof("To check status, scale up the required deployments and cleanup later, run:") log.Infof(" sts-backup %s check-and-finalize --job %s --wait -n %s", serviceName, jobName, namespace) diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index c33d4bc..05482b3 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -17,7 +17,7 @@ download_from_s3() { local dest="$3" local backup_file="$4" echo "=== Downloading Settings backup \"${backup_file}\" from bucket \"${bucket}\"..." - sts-toolbox aws s3 --endpoint "http://${MINIO_ENDPOINT}" --region minio cp "s3://${bucket}/${prefix}${backup_file}" "${dest}/${backup_file}" + sts-toolbox aws s3 --endpoint "http://${S3_ENDPOINT}" --region us-east-1 cp "s3://${bucket}/${prefix}${backup_file}" "${dest}/${backup_file}" } RESTORE_FILE="" diff --git a/internal/scripts/scripts/restore-stackgraph-backup-v2-abort.sh b/internal/scripts/scripts/restore-stackgraph-backup-v2-abort.sh new file mode 100644 index 0000000..6664a55 --- /dev/null +++ b/internal/scripts/scripts/restore-stackgraph-backup-v2-abort.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source $SCRIPT_DIR/restore-stackgraph-backup-v2-env.sh + +echo "=== Finalizing historic StackGraph backup data (v2)" + +/opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import-v2-abort "s3a://${BACKUP_V2_LOCATION}" +echo "=== StackGraph restore finalized" + + diff --git a/internal/scripts/scripts/restore-stackgraph-backup-v2-backfill.sh b/internal/scripts/scripts/restore-stackgraph-backup-v2-backfill.sh new file mode 100644 index 0000000..5332f9f --- /dev/null +++ b/internal/scripts/scripts/restore-stackgraph-backup-v2-backfill.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source $SCRIPT_DIR/restore-stackgraph-backup-v2-env.sh + +echo "=== Backfilling historic StackGraph backup data (v2)" + +/opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import-v2-backfill "s3a://${BACKUP_V2_LOCATION}" +echo "=== StackGraph restore complete" + + diff --git a/internal/scripts/scripts/restore-stackgraph-backup-v2-env.sh b/internal/scripts/scripts/restore-stackgraph-backup-v2-env.sh new file mode 100644 index 0000000..e4c0130 --- /dev/null +++ b/internal/scripts/scripts/restore-stackgraph-backup-v2-env.sh @@ -0,0 +1,30 @@ +export AWS_ACCESS_KEY_ID +AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" +export AWS_SECRET_ACCESS_KEY +AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" + +export BACKUP_V2_LOCATION="${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}v2/" + +TYPESAFE_ESCAPED_BUCKET=$(echo "${BACKUP_STACKGRAPH_BUCKET_NAME}" | sed 's/_/___/g; s/-/__/g; s/\./_/g') +# HACK: We configure hbase here through typesafe config. However, typesafe does not support a key being both object and +# string (as in `endpoint = "string"` and `endpoint.region = "us-east-1"` +# We build a little hack there, which allows postfixing endpoint as endpoint__, which gets stripped when transforming typesafe to hbase conf +AWS_BUCKET_ENDPOINT_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_endpoint___" +export "${AWS_BUCKET_ENDPOINT_VAR}=http://${S3_ENDPOINT}" +AWS_BUCKET_REGION_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_endpoint_region" +export "${AWS_BUCKET_REGION_VAR}=us-east-1" +AWS_BUCKET_ACCESS_KEY_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_access_key" +export "${AWS_BUCKET_ACCESS_KEY_VAR}=$(cat /aws-keys/accesskey)" +AWS_BUCKET_SECRET_KEY_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_secret_key" +export "${AWS_BUCKET_SECRET_KEY_VAR}=$(cat /aws-keys/secretkey)" + +AWS_BUCKET_PATH_STYLE_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_path_style_access" +export "${AWS_BUCKET_PATH_STYLE_VAR}=true" +AWS_BUCKET_CONNECTION_SSL_ENABLED_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_connection_ssl_enabled" +export "${AWS_BUCKET_CONNECTION_SSL_ENABLED_VAR}=false" +AWS_BUCKET_AWS_CREDENTIALS_PROVIDER_VAR="CONFIG_FORCE_fs_s3a_bucket_${TYPESAFE_ESCAPED_BUCKET}_aws_credentials_provider" +export "${AWS_BUCKET_AWS_CREDENTIALS_PROVIDER_VAR}=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + +# Increase the hbase client timeout for bulk operations. +export CONFIG_FORCE_hbase_rpc_timeout="120000" + diff --git a/internal/scripts/scripts/restore-stackgraph-backup-v2-live.sh b/internal/scripts/scripts/restore-stackgraph-backup-v2-live.sh new file mode 100644 index 0000000..8dd736a --- /dev/null +++ b/internal/scripts/scripts/restore-stackgraph-backup-v2-live.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +TMP_DIR=/tmp-data + +source $SCRIPT_DIR/restore-stackgraph-backup-v2-env.sh + +echo "=== Importing StackGraph data (v2) from \"${BACKUP_FILE}\"..." + +/opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import-v2-live "s3a://${BACKUP_V2_LOCATION}" -backup-name "${BACKUP_FILE}" "${FORCE_DELETE}" + +echo "=== StackGraph live data loaded, please continue with backfill" + +# === StackPacks Restore === +if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then + echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" +else + # Construct stackpacks backup filename from the original backup file + STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" + + echo "=== Checking for StackPacks backup (v2) \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." + + # Check if stackpacks backup exists in S3 + if ! sts-toolbox aws s3 ls --endpoint "http://${S3_ENDPOINT}" --region us-east-1 --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_S3_PREFIX}v2/${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== ERROR: StackPacks backup \"${STACKPACKS_FILE}\" not found in S3" + exit 1 + fi + + echo "=== Downloading StackPacks backup..." + sts-toolbox aws s3 cp --endpoint "http://${S3_ENDPOINT}" --region us-east-1 "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}v2/${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}" + echo "=== StackPacks restore complete" +fi +echo "===" diff --git a/internal/scripts/scripts/restore-stackgraph-backup.sh b/internal/scripts/scripts/restore-stackgraph-backup.sh index 13b1894..7d5b2ff 100644 --- a/internal/scripts/scripts/restore-stackgraph-backup.sh +++ b/internal/scripts/scripts/restore-stackgraph-backup.sh @@ -9,7 +9,7 @@ export AWS_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" echo "=== Downloading StackGraph backup \"${BACKUP_FILE}\" from bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." -sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" +sts-toolbox aws s3 cp --endpoint "http://${S3_ENDPOINT}" --region us-east-1 "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" echo "=== Importing StackGraph data from \"${BACKUP_FILE}\"..." /opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import "${TMP_DIR}/${BACKUP_FILE}" "${FORCE_DELETE}" @@ -25,13 +25,13 @@ else echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." # Check if stackpacks backup exists in S3 - if ! sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + if ! sts-toolbox aws s3 ls --endpoint "http://${S3_ENDPOINT}" --region us-east-1 --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then echo "=== ERROR: StackPacks backup \"${STACKPACKS_FILE}\" not found in S3" exit 1 fi echo "=== Downloading StackPacks backup..." - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + sts-toolbox aws s3 cp --endpoint "http://${S3_ENDPOINT}" --region us-east-1 "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}" diff --git a/internal/scripts/scripts/restore-victoria-metrics-backup.sh b/internal/scripts/scripts/restore-victoria-metrics-backup.sh index ee2e4f1..ceb3d34 100644 --- a/internal/scripts/scripts/restore-victoria-metrics-backup.sh +++ b/internal/scripts/scripts/restore-victoria-metrics-backup.sh @@ -14,4 +14,4 @@ AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" export AWS_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" -/vmrestore-prod -storageDataPath=/storage -src="s3://$S3_LOCATION" -customS3Endpoint="http://$MINIO_ENDPOINT" -httpListenAddr "$METRICS_ADDR" +/vmrestore-prod -storageDataPath=/storage -src="s3://$S3_LOCATION" -customS3Endpoint="http://$S3_ENDPOINT" -httpListenAddr "$METRICS_ADDR" From 1a76699f12b2d7de236e00078f13be8b9264af11 Mon Sep 17 00:00:00 2001 From: Bram Schuur Date: Mon, 20 Jul 2026 09:44:34 +0200 Subject: [PATCH 2/3] STAC-24630: Work out review comments --- cmd/stackgraphv2/backfill.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/stackgraphv2/backfill.go b/cmd/stackgraphv2/backfill.go index f0a8ff9..6ec8566 100644 --- a/cmd/stackgraphv2/backfill.go +++ b/cmd/stackgraphv2/backfill.go @@ -21,9 +21,11 @@ const ( func backfillCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "backfill", - Short: "Complete a restore by backfilling old data", - Long: "Complete a restore by backfilling old data. This process can run while the system is already up and running " + - "After the backfill is done, the restore process is complete. ", + Short: "Complete an interrupted restore by backfilling old data", + Long: "Complete an interrupted restore by backfilling old data. This process can run while the system is already up and running." + + "After the backfill is done, the restore process is complete. " + + "During a normal restore the restore command will take care of backfilling the data, but if that gets interrupted (Ctrl-C) " + + "or somehow crashes due to cluster instability, this command allows retrying the backfill portion of the restore.", Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runBackfill, cmdutils.StorageIsRequired) }, From 24b1933ef3ebda30b8892cef551c156a434ea391 Mon Sep 17 00:00:00 2001 From: Bram Schuur Date: Tue, 21 Jul 2026 16:31:32 +0200 Subject: [PATCH 3/3] STAC-24630: Add additional message when jobs fail/get interrupted --- cmd/stackgraphv2/abort.go | 11 ++++-- cmd/stackgraphv2/backfill.go | 5 ++- cmd/stackgraphv2/check_and_finalize.go | 12 ++++--- cmd/stackgraphv2/logging.go | 42 ++++++++++++++++++++++ cmd/stackgraphv2/restore.go | 5 +-- internal/orchestration/restore/finalize.go | 1 + 6 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 cmd/stackgraphv2/logging.go diff --git a/cmd/stackgraphv2/abort.go b/cmd/stackgraphv2/abort.go index 24fa11f..492a210 100644 --- a/cmd/stackgraphv2/abort.go +++ b/cmd/stackgraphv2/abort.go @@ -14,7 +14,7 @@ import ( ) const ( - abortNameTemplate = "stackgraph-abort-v2" + abortNameTemplate = "stackgraph-v2-abort" abortScript = "/backup-restore-scripts/restore-stackgraph-backup-v2-abort.sh" ) @@ -52,7 +52,14 @@ func runAbort(appCtx *app.Context) error { appCtx.Logger.Successf("Abort job created: %s", jobName) - return waitAndCleanupAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) + err := waitAndCleanupAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) + + if err != nil { + logAfterJobResult(appCtx.Logger, checkJobName, false) + return err + } + + return nil } // waitAndCleanupAbortJob waits for job completion and cleans up resources diff --git a/cmd/stackgraphv2/backfill.go b/cmd/stackgraphv2/backfill.go index 6ec8566..c5eaa8d 100644 --- a/cmd/stackgraphv2/backfill.go +++ b/cmd/stackgraphv2/backfill.go @@ -14,7 +14,7 @@ import ( ) const ( - backfillNameTemplate = "stackgraph-backfill-v2" + backfillNameTemplate = "stackgraph-v2-backfill" backfillScript = "/backup-restore-scripts/restore-stackgraph-backup-v2-backfill.sh" ) @@ -56,8 +56,7 @@ func runBackfill(appCtx *app.Context) error { err := waitAndCleanupBackfillJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) if err != nil { - appCtx.Logger.Println() - appCtx.Logger.Infof("Backfill failed. It is possible to restart the backfill without starting a complete restore.") + logAfterJobResult(appCtx.Logger, checkJobName, false) return err } diff --git a/cmd/stackgraphv2/check_and_finalize.go b/cmd/stackgraphv2/check_and_finalize.go index 39cd794..9f266fe 100644 --- a/cmd/stackgraphv2/check_and_finalize.go +++ b/cmd/stackgraphv2/check_and_finalize.go @@ -21,15 +21,15 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Check and finalize a Stackgraph restore (v2) job", Long: `Check the status of a background Stackgraph restore job and clean up resources. -This command is useful when a restore job was started with --background flag or was interrupted (Ctrl+C). +This command is useful when a restore/backfill/abort job was interrupted (Ctrl+C). It will check the job status, print logs if it failed, and clean up the job and PVC resources. Examples: # Check job status without waiting - sts-backup stackgraph-v2 check-and-finalize --job stackgraph-restore-20250128t143000 -n my-namespace + sts-backup stackgraph-v2 check-and-finalize --job stackgraph-v2-restore-20250128t143000 -n my-namespace # Wait for job completion and cleanup - sts-backup stackgraph-v2 check-and-finalize --job stackgraph-restore-20250128t143000 --wait -n my-namespace`, + sts-backup stackgraph-v2 check-and-finalize --job stackgraph-v2-restore-20250128t143000 --wait -n my-namespace`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired) }, @@ -43,11 +43,11 @@ Examples: } func runCheckAndFinalize(appCtx *app.Context) error { - return restore.CheckAndFinalize(restore.CheckAndFinalizeParams{ + err := restore.CheckAndFinalize(restore.CheckAndFinalizeParams{ K8sClient: appCtx.K8sClient, Namespace: appCtx.Namespace, JobName: checkJobName, - ServiceName: "stackgraph", + ServiceName: "stackgraph-v2", ScaleUpFn: scale.ScaleUpAndReleaseLock, ScaleDownFn: scale.ScaleDown, ScaleSelector: appCtx.Config.Stackgraph.Restore.ScaleDownLabelSelector, @@ -55,4 +55,6 @@ func runCheckAndFinalize(appCtx *app.Context) error { WaitForJob: waitForJob, Log: appCtx.Logger, }) + logAfterJobResult(appCtx.Logger, checkJobName, err == nil) + return err } diff --git a/cmd/stackgraphv2/logging.go b/cmd/stackgraphv2/logging.go new file mode 100644 index 0000000..dfbca44 --- /dev/null +++ b/cmd/stackgraphv2/logging.go @@ -0,0 +1,42 @@ +package stackgraphv2 + +import ( + "strings" + + "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" +) + +func logAfterJobResult(log *logger.Logger, jobName string, success bool) { + switch { + case strings.HasPrefix(jobName, restoreNameTemplate): + if success { + log.Println() + log.Infof("Job '%s' has successfully restored the live data. The system is running again.", jobName) + log.Infof("Now run `sts-backup stackgraph-v2 backfill` to load the historic data.") + } else { + log.Println() + log.Infof("Job '%s' has failed to restore the live data. The system was brought back up but the data was not restored.", jobName) + log.Infof("Rerun your `sts-backup stackgraph-v2 restore ...` command toretry restoring data.") + } + case strings.HasPrefix(jobName, backfillNameTemplate): + if success { + log.Println() + log.Infof("Job '%s' has successfully backfilled the historic data. The restore is complete.", jobName) + } else { + log.Println() + log.Infof("Job '%s' has failed to backfill all historic data.", jobName) + log.Infof("Run `sts-backup stackgraph-v2 backfill` to retry restoring old data, or") + log.Infof("run `sts-backup stackgraph-v2 abort` to leave the restored data as-is and continue with") + log.Infof("the data currently in the system.") + } + case strings.HasPrefix(jobName, abortNameTemplate): + if success { + log.Println() + log.Infof("Job '%s' has successfully aborted. Historic data is incomplete but the system is functional.", jobName) + } else { + log.Println() + log.Infof("Job '%s' has failed to abort the restore.", jobName) + log.Infof("Rerun `sts-backup stackgraph-v2 abort` again to try again.") + } + } +} diff --git a/cmd/stackgraphv2/restore.go b/cmd/stackgraphv2/restore.go index c42ec8d..e03a2d3 100644 --- a/cmd/stackgraphv2/restore.go +++ b/cmd/stackgraphv2/restore.go @@ -24,7 +24,7 @@ import ( ) const ( - jobNameTemplate = "stackgraph-restore-v2" + restoreNameTemplate = "stackgraph-v2-restore" configMapDefaultFileMode = 0755 purgeStackgraphDataFlag = "-force" ) @@ -136,7 +136,7 @@ func liveRestore(appCtx *app.Context) error { appCtx.Logger.Println() appCtx.Logger.Infof("Creating restore of live data job for backup: %s", backupFile) - jobName := fmt.Sprintf("%s-%s", jobNameTemplate, time.Now().Format("20060102t150405")) + jobName := fmt.Sprintf("%s-%s", restoreNameTemplate, time.Now().Format("20060102t150405")) if err = createRestoreJob(appCtx.K8sClient, appCtx.Namespace, jobName, backupFile, appCtx.Config); err != nil { return fmt.Errorf("failed to create restore job: %w", err) @@ -146,6 +146,7 @@ func liveRestore(appCtx *app.Context) error { err = waitAndCleanupRestoreJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger) if err != nil { + logAfterJobResult(appCtx.Logger, checkJobName, false) return err } diff --git a/internal/orchestration/restore/finalize.go b/internal/orchestration/restore/finalize.go index ae3ba1f..a737b0e 100644 --- a/internal/orchestration/restore/finalize.go +++ b/internal/orchestration/restore/finalize.go @@ -117,6 +117,7 @@ type CheckAndFinalizeParams struct { // CheckAndFinalize checks the status of a background restore job and cleans up resources // This is useful when a restore job was started with --background flag or was interrupted (Ctrl+C) +// Returns whether the job succeeded func CheckAndFinalize(params CheckAndFinalizeParams) error { // Get job params.Log.Infof("Checking status of job: %s", params.JobName)