From d4fc02ec15c71116cb534762117b1a8dee240196 Mon Sep 17 00:00:00 2001 From: ercross Date: Mon, 12 Oct 2026 11:29:41 +0100 Subject: [PATCH] bound forward and rollback deployment attempts --- CHANGELOG.md | 37 +++ README.md | 42 ++++ config/config.go | 113 ++++++++- config/config_test.go | 117 +++++++++ deployment/deployer.go | 2 +- deployment/docker_deployer.go | 243 +++++++++++++++++-- deployment/docker_deployer_test.go | 368 +++++++++++++++++++++++++++-- notifier/notifier.go | 4 +- notifier/slack_notifier.go | 8 +- reconciler/reconciler.go | 65 ++++- reconciler/reconciler_test.go | 131 +++++++++- state/local_fs.go | 112 +++++++-- state/local_fs_test.go | 124 +++++++++- state/state.go | 39 ++- 14 files changed, 1307 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b8ae63..116856d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## v0.4.2 - 2026-10-12 + +### Added + +- Added `deployment_retry` configuration for bounded forward deployment and rollback attempts. +- Added top-level `deployment_retry` defaults that apply to every workload. +- Added workload-level `deployment_retry` overrides; workload settings override only the fields they explicitly set. +- Added exponential retry backoff with jitter for deployment strategy, deploy command, health-check, and rollback failures. +- Added persisted terminal failure state for deployment digests that exhaust forward retry attempts, preventing repeated automatic deployment loops. + +### Changed + +- Forward deployment attempts default to `1`, preserving previous single-attempt behavior unless retry settings are configured. +- Rollback attempts default to `1`, with retry delay defaults of `1s` initial interval, `30s` max interval, and multiplier `2`. +- Rollback now starts only after forward deployment attempts are exhausted. +- Reconciliation now skips automatic deployment when a terminal failure matches the desired manifest digest. +- Reconciliation allows deployment of a newer desired manifest digest when an existing terminal failure belongs to an older digest. +- Successful forward deployment commits now clear terminal failure state for the workload. +- Successful rollback after forward retry exhaustion now persists terminal failure for the failed desired digest, preventing the next reconciliation cycle from retrying the same known-bad digest automatically. +- Deployment state types were clarified: `DeploymentState` is the outer per-workload record, while `DeploymentSnapshot` is the current or previous deployed snapshot. +- Slack success notifications now receive workload name separately from the deployment snapshot. + +### Fixed + +- Context cancellation during deployment retry backoff and rollback retry backoff now stops retries promptly. +- Unexpected previous-state load errors now abort deployment instead of attempting rollback with an empty previous snapshot. +- Unexpected terminal-failure load errors now abort reconciliation instead of treating terminal failure state as absent. +- Terminal failure records for deployments with no previous state now record zero rollback attempts because no rollback was attempted. +- Rollback commits no longer clear terminal failure state, so a failed desired digest remains blocked even after rollback restores the previous known-good deployment. +- Commit failures during deployment or rollback now surface as deployment errors instead of being reported as successful deployments. +- Backoff delay calculation now clamps safely before duration conversion and caps jittered delays to the configured max interval. + +### Upgrade Notes + +- Existing central state files remain compatible with the new state layout. Terminal failure state is stored as an optional `terminal_failure` field and is omitted when absent. +- `DeploymentSnapshot` and `TerminalFailureState` no longer carry `workload_name`; the outer `DeploymentState.workload_name` identifies the workload. + ## v0.4.1 - 2026-10-08 ### Changed diff --git a/README.md b/README.md index e44b2f5..0d17a15 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,13 @@ contains one workload. Minimal example: ```yaml +deployment_retry: + max_retry_forward_deployment: 3 + max_retry_rollback: 2 + initial_interval: 1s + max_interval: 30s + multiplier: 2 + workloads: - name: my-app release_prefix: my-app-prod @@ -168,6 +175,36 @@ health_check: timeout: 5s ``` +## Deployment Retries + +By default, `reconD` keeps the original single-attempt behavior: + +```yaml +deployment_retry: + max_retry_forward_deployment: 1 + max_retry_rollback: 1 + initial_interval: 1s + max_interval: 30s + multiplier: 2 +``` + +Increase `max_retry_forward_deployment` to retry transient strategy, +`deploy_command`, or health-check failures before rollback. Rollback starts only +after the forward deployment attempts are exhausted. Increase +`max_retry_rollback` to retry rollback failures. Retry waits use exponential +backoff with jitter and stop promptly when the agent context is canceled. + +Set `deployment_retry` at the top level to define defaults for every workload in +the config. A workload can include its own `deployment_retry` section to +override only the fields that differ from the top-level defaults. + +If forward deployment attempts are exhausted, `reconD` persists a terminal +failure for that workload and manifest digest in the central state file. When a +previous deployment exists, rollback still runs to recover the workload, but a +successful rollback does not make the failed desired digest eligible for +automatic retry. Future reconciliation cycles skip that same digest until the +terminal failure is cleared from state. A newer desired digest is not blocked. + ## State Files State for all workloads is stored in one deployments state file. Set it with @@ -288,6 +325,11 @@ Common optional fields: - `health_check.interval`: defaults to `10s`. - `health_check.timeout`: defaults to `5s`. - `labels`: labels used to scope Docker image pruning. +- `deployment_retry.max_retry_forward_deployment`: total forward deployment attempts before rollback; defaults to `1`. +- `deployment_retry.max_retry_rollback`: total rollback recovery attempts after forward deployment is exhausted; defaults to `1`. +- `deployment_retry.initial_interval`: first retry backoff delay; defaults to `1s`. +- `deployment_retry.max_interval`: maximum retry backoff delay; defaults to `30s`. +- `deployment_retry.multiplier`: exponential backoff multiplier; defaults to `2`. ## Design Rule diff --git a/config/config.go b/config/config.go index 820da41..7617fd3 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ package config import ( "errors" "fmt" + "math" "os" "path/filepath" "strings" @@ -16,6 +17,10 @@ type Config struct { // Log is optional and controls reconD's process-level logging output. Log Log `yaml:"log"` + // DeploymentRetry is optional and provides default retry behavior for every + // workload in this config. Workload-level deployment_retry fields override it. + DeploymentRetry DeploymentRetry `yaml:"deployment_retry"` + // Workloads is the required list of container workloads reconD should watch // and reconcile. Configure one entry for each independently deployed // service. @@ -77,6 +82,10 @@ type Workload struct { // image tag. Strategy Strategy `yaml:"strategy"` + // DeploymentRetry bounds automatic deployment and rollback attempts for one + // desired deployment identity. + DeploymentRetry DeploymentRetry `yaml:"deployment_retry"` + // Labels is optional and scopes Docker image pruning to images with these // labels after a successful deployment. Labels map[string]string `yaml:"labels"` @@ -112,6 +121,25 @@ type Strategy struct { ImageTagKey string `yaml:"image_tag_key"` } +type DeploymentRetry struct { + // MaxRetryForwardDeployment is the total number of forward deployment + // attempts before rollback is considered. Defaults to 1. + MaxRetryForwardDeployment int `yaml:"max_retry_forward_deployment"` + + // MaxRetryRollback is the total number of rollback attempts before the + // desired deployment is marked terminally failed. Defaults to 1. + MaxRetryRollback int `yaml:"max_retry_rollback"` + + // InitialInterval is the first delay before retrying. Defaults to 1s. + InitialInterval Duration `yaml:"initial_interval"` + + // MaxInterval caps exponential backoff delays. Defaults to 30s. + MaxInterval Duration `yaml:"max_interval"` + + // Multiplier controls exponential backoff growth. Defaults to 2. + Multiplier float64 `yaml:"multiplier"` +} + type StrategyType string const ( @@ -165,7 +193,9 @@ func Load(path string) (*Config, error) { return nil, err } + cfg.DeploymentRetry.fillDefaultOnZeroValues() for i := range cfg.Workloads { + cfg.Workloads[i].DeploymentRetry = mergeDeploymentRetry(cfg.DeploymentRetry, cfg.Workloads[i].DeploymentRetry) cfg.Workloads[i].fillDefaultOnZeroValues() } @@ -212,12 +242,68 @@ func (w *Workload) fillDefaultOnZeroValues() { if w.Strategy.Type == StrategyEnvFile && w.Strategy.ImageTagKey == "" { w.Strategy.ImageTagKey = "IMAGE_TAG" } + if w.DeploymentRetry.MaxRetryForwardDeployment == 0 { + w.DeploymentRetry.MaxRetryForwardDeployment = 1 + } + if w.DeploymentRetry.MaxRetryRollback == 0 { + w.DeploymentRetry.MaxRetryRollback = 1 + } + if w.DeploymentRetry.InitialInterval.Duration == 0 { + w.DeploymentRetry.InitialInterval.Duration = time.Second + } + if w.DeploymentRetry.MaxInterval.Duration == 0 { + w.DeploymentRetry.MaxInterval.Duration = 30 * time.Second + } + if w.DeploymentRetry.Multiplier == 0 { + w.DeploymentRetry.Multiplier = 2 + } +} + +func (r *DeploymentRetry) fillDefaultOnZeroValues() { + if r.MaxRetryForwardDeployment == 0 { + r.MaxRetryForwardDeployment = 1 + } + if r.MaxRetryRollback == 0 { + r.MaxRetryRollback = 1 + } + if r.InitialInterval.Duration == 0 { + r.InitialInterval.Duration = time.Second + } + if r.MaxInterval.Duration == 0 { + r.MaxInterval.Duration = 30 * time.Second + } + if r.Multiplier == 0 { + r.Multiplier = 2 + } +} + +func mergeDeploymentRetry(defaults, override DeploymentRetry) DeploymentRetry { + merged := defaults + if override.MaxRetryForwardDeployment != 0 { + merged.MaxRetryForwardDeployment = override.MaxRetryForwardDeployment + } + if override.MaxRetryRollback != 0 { + merged.MaxRetryRollback = override.MaxRetryRollback + } + if override.InitialInterval.Duration != 0 { + merged.InitialInterval = override.InitialInterval + } + if override.MaxInterval.Duration != 0 { + merged.MaxInterval = override.MaxInterval + } + if override.Multiplier != 0 { + merged.Multiplier = override.Multiplier + } + return merged } func (cfg Config) validate() error { if len(cfg.Workloads) == 0 { return fmt.Errorf("at least one workload must be configured") } + if err := cfg.DeploymentRetry.validate(); err != nil { + return err + } workloadNames := make(map[string]struct{}, len(cfg.Workloads)) for i, workload := range cfg.Workloads { @@ -251,7 +337,10 @@ func (w Workload) validate() error { if err := w.HealthCheck.validate(); err != nil { return err } - return w.Strategy.validate() + if err := w.Strategy.validate(); err != nil { + return err + } + return w.DeploymentRetry.validate() } func (h HealthCheck) validate() error { @@ -287,6 +376,28 @@ func (s Strategy) validate() error { return nil } +func (r DeploymentRetry) validate() error { + if r.MaxRetryForwardDeployment < 1 { + return errors.New("deployment_retry.max_retry_forward_deployment must be at least 1") + } + if r.MaxRetryRollback < 1 { + return errors.New("deployment_retry.max_retry_rollback must be at least 1") + } + if r.InitialInterval.Duration <= 0 { + return errors.New("deployment_retry.initial_interval must be greater than 0") + } + if r.MaxInterval.Duration <= 0 { + return errors.New("deployment_retry.max_interval must be greater than 0") + } + if r.MaxInterval.Duration < r.InitialInterval.Duration { + return errors.New("deployment_retry.max_interval must be greater than or equal to initial_interval") + } + if math.IsNaN(r.Multiplier) || math.IsInf(r.Multiplier, 0) || r.Multiplier < 1 { + return errors.New("deployment_retry.multiplier must be a finite number at least 1") + } + return nil +} + func (d *Duration) UnmarshalYAML(value *yaml.Node) error { if value.ShortTag() == "!!str" { var s string diff --git a/config/config_test.go b/config/config_test.go index 6fde8a3..5cf6d32 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "math" "os" "path/filepath" "testing" @@ -77,6 +78,122 @@ workloads: require.Equal(t, 2*time.Minute, cfg.Workloads[1].CheckInterval.Duration) } +func TestLoadDeploymentRetryConfig(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "workloads.yml") + require.NoError(t, os.WriteFile(path, []byte(` +workloads: + - name: api + container_name: api + deploy_command: docker restart api + git_provider: + owner: toughbred + repo: gymportal + deployment_retry: + max_retry_forward_deployment: 3 + max_retry_rollback: 2 + initial_interval: 100ms + max_interval: 1s + multiplier: 1.5 +`), 0o600)) + + cfg, err := Load(path) + require.NoError(t, err) + require.Equal(t, 3, cfg.Workloads[0].DeploymentRetry.MaxRetryForwardDeployment) + require.Equal(t, 2, cfg.Workloads[0].DeploymentRetry.MaxRetryRollback) + require.Equal(t, 100*time.Millisecond, cfg.Workloads[0].DeploymentRetry.InitialInterval.Duration) + require.Equal(t, time.Second, cfg.Workloads[0].DeploymentRetry.MaxInterval.Duration) + require.Equal(t, 1.5, cfg.Workloads[0].DeploymentRetry.Multiplier) +} + +func TestLoadDefaultsDeploymentRetryConfig(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "workloads.yml") + require.NoError(t, os.WriteFile(path, []byte(` +workloads: + - name: api + container_name: api + deploy_command: docker restart api + git_provider: + owner: toughbred + repo: gymportal +`), 0o600)) + + cfg, err := Load(path) + require.NoError(t, err) + require.Equal(t, 1, cfg.Workloads[0].DeploymentRetry.MaxRetryForwardDeployment) + require.Equal(t, 1, cfg.Workloads[0].DeploymentRetry.MaxRetryRollback) + require.Equal(t, time.Second, cfg.Workloads[0].DeploymentRetry.InitialInterval.Duration) + require.Equal(t, 30*time.Second, cfg.Workloads[0].DeploymentRetry.MaxInterval.Duration) + require.Equal(t, float64(2), cfg.Workloads[0].DeploymentRetry.Multiplier) +} + +func TestLoadTopLevelDeploymentRetryDefaultsApplyToWorkloads(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "workloads.yml") + require.NoError(t, os.WriteFile(path, []byte(` +deployment_retry: + max_retry_forward_deployment: 4 + max_retry_rollback: 3 + initial_interval: 250ms + max_interval: 5s + multiplier: 1.25 +workloads: + - name: api + container_name: api + deploy_command: docker restart api + git_provider: + owner: toughbred + repo: gymportal + - name: worker + container_name: worker + deploy_command: docker restart worker + git_provider: + owner: toughbred + repo: gymportal + deployment_retry: + max_retry_forward_deployment: 2 + initial_interval: 100ms +`), 0o600)) + + cfg, err := Load(path) + require.NoError(t, err) + + require.Equal(t, 4, cfg.DeploymentRetry.MaxRetryForwardDeployment) + require.Equal(t, 3, cfg.DeploymentRetry.MaxRetryRollback) + require.Equal(t, 250*time.Millisecond, cfg.DeploymentRetry.InitialInterval.Duration) + require.Equal(t, 5*time.Second, cfg.DeploymentRetry.MaxInterval.Duration) + require.Equal(t, 1.25, cfg.DeploymentRetry.Multiplier) + + require.Equal(t, cfg.DeploymentRetry, cfg.Workloads[0].DeploymentRetry) + require.Equal(t, 2, cfg.Workloads[1].DeploymentRetry.MaxRetryForwardDeployment) + require.Equal(t, 3, cfg.Workloads[1].DeploymentRetry.MaxRetryRollback) + require.Equal(t, 100*time.Millisecond, cfg.Workloads[1].DeploymentRetry.InitialInterval.Duration) + require.Equal(t, 5*time.Second, cfg.Workloads[1].DeploymentRetry.MaxInterval.Duration) + require.Equal(t, 1.25, cfg.Workloads[1].DeploymentRetry.Multiplier) +} + +func TestDeploymentRetryValidateRejectsNonFiniteMultiplier(t *testing.T) { + t.Parallel() + + base := DeploymentRetry{ + MaxRetryForwardDeployment: 1, + MaxRetryRollback: 1, + InitialInterval: Duration{Duration: time.Second}, + MaxInterval: Duration{Duration: time.Second}, + Multiplier: 2, + } + + for _, multiplier := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + retry := base + retry.Multiplier = multiplier + require.ErrorContains(t, retry.validate(), "finite number") + } +} + func TestLoadRejectsDuplicateWorkloadName(t *testing.T) { t.Parallel() diff --git a/deployment/deployer.go b/deployment/deployer.go index d3acda0..7636c5d 100644 --- a/deployment/deployer.go +++ b/deployment/deployer.go @@ -25,7 +25,7 @@ var ( ) var errorsThatCanTriggerRollback = []error{ - errImagePullFailed, errDeployCommandFailed, errHealthCheckFailed, + errImagePullFailed, errDeploymentStrategyFailed, errDeployCommandFailed, errHealthCheckFailed, } // RuntimeState is the reconciler's view of the currently running workload. diff --git a/deployment/docker_deployer.go b/deployment/docker_deployer.go index 3c87bdc..eeff9a8 100644 --- a/deployment/docker_deployer.go +++ b/deployment/docker_deployer.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "math/rand" "os" "os/exec" "strings" @@ -149,60 +150,251 @@ func (d *dockerDeployer) Deploy(ctx context.Context, workload config.Workload, m previousState, err := stateMgr.LoadDeployed(workload.Name) if err != nil && !state.IsNotFound(err) { - d.logger.Warn("could not load previous state", "error", err) + return fmt.Errorf("could not load previous state: %w", err) } noPreviousState := state.IsNotFound(err) d.notifier.NotifyOnNewDeploymentStarted(workload.Name, meta) - newState, err := d.deploy(ctx, workload, meta, stateMgr) + newState, err := d.deployWithRetry(ctx, workload, meta, stateMgr) if err == nil { - d.notifier.NotifyOnDeploymentSuccess(newState) + d.notifier.NotifyOnDeploymentSuccess(workload.Name, newState) if err = d.PruneDanglingImages(ctx, workload.Labels); err != nil { d.logger.Error("failed to prune dangling images", "phase", logger.PhaseCommit, "workload", workload.Name, "error", err) } return nil } - d.notifier.NotifyOnDeploymentFailed(workload.Name, meta, err) - d.logger.Error("deployment failed", "workload", workload.Name, "error", err) + d.logger.Error("forward deployment failed", "workload", workload.Name, "error", err) + if contextIsDone(err) { + return err + } if !errorShouldTriggerRollback(err) { + d.notifier.NotifyOnDeploymentFailed(workload.Name, meta, err) return fmt.Errorf("deployment failed but no rollback required: %w", err) } if noPreviousState { d.logger.Error("no previous state available for rollback", "phase", logger.PhaseRollback) - return fmt.Errorf("rollback failed: no previous deployment state") + d.notifier.NotifyOnDeploymentFailed(workload.Name, meta, err) + if terminalErr := d.commitTerminalFailureOnNoPreviousStateAvailable(stateMgr, workload, meta, err); terminalErr != nil { + return fmt.Errorf("deployment failed; failed to persist no-previous-state terminal failure: %v", terminalErr) + } + return fmt.Errorf("deployment failed: no previous deployment state; no rollback attempted") } - rollbackState, rollbackErr := d.rollback(ctx, stateMgr, workload, previousState, meta.ManifestDigest, err.Error()) + d.notifier.NotifyOnDeploymentFailed(workload.Name, meta, err) + rollbackState, rollbackAttempts, rollbackErr := d.rollbackWithRetry(ctx, stateMgr, workload, previousState, meta, err.Error()) if rollbackErr == nil { - d.notifier.NotifyOnDeploymentSuccess(rollbackState) - return nil + d.notifier.NotifyOnDeploymentSuccess(workload.Name, rollbackState) + if terminalErr := d.commitTerminalFailure(stateMgr, workload, meta, err, nil, rollbackAttempts); terminalErr != nil { + return fmt.Errorf("deployment failed and rollback succeeded; additionally failed to persist terminal failure: %v", terminalErr) + } + return fmt.Errorf("deployment failed after %d forward attempts; rollback succeeded after %d attempts", deploymentRetryPolicy(workload).MaxRetryForwardDeployment, rollbackAttempts) + } + if contextIsDone(rollbackErr) { + return rollbackErr } d.notifier.NotifyOnDeploymentFailed(workload.Name, meta, fmt.Errorf("ROLLBACK FAILED: %w", rollbackErr)) + if terminalErr := d.commitTerminalFailure(stateMgr, workload, meta, err, rollbackErr, rollbackAttempts); terminalErr != nil { + return fmt.Errorf("deployment and rollback failed: %w; additionally failed to persist terminal failure: %v", rollbackErr, terminalErr) + } return fmt.Errorf("deployment and rollback failed: %w", rollbackErr) } -func (d *dockerDeployer) deploy(ctx context.Context, workload config.Workload, meta git_provider.DeploymentMetadata, stateMgr state.Manager) (state.DeploymentState, error) { - var newState state.DeploymentState +func (d *dockerDeployer) deployWithRetry(ctx context.Context, workload config.Workload, meta git_provider.DeploymentMetadata, stateMgr state.Manager) (state.DeploymentSnapshot, error) { + policy := deploymentRetryPolicy(workload) + var lastErr error + + for attempt := 1; attempt <= policy.MaxRetryForwardDeployment; attempt++ { + d.logger.Info("running deployment attempt", + "phase", logger.PhaseRestart, + "workload", workload.Name, + "attempt", attempt, + "max_attempts", policy.MaxRetryForwardDeployment, + ) + + newState, err := d.deploy(ctx, workload, meta, stateMgr) + if err == nil { + return newState, nil + } + if contextIsDone(err) { + return state.DeploymentSnapshot{}, err + } + lastErr = err + if !errorShouldTriggerRollback(err) || attempt == policy.MaxRetryForwardDeployment { + break + } + if err := d.sleepBeforeRetry(ctx, policy, attempt, "forward_deployment", workload.Name, err); err != nil { + return state.DeploymentSnapshot{}, err + } + } + + return state.DeploymentSnapshot{}, lastErr +} + +func (d *dockerDeployer) rollbackWithRetry(ctx context.Context, stateMgr state.Manager, workload config.Workload, previousState state.DeploymentSnapshot, failedMeta git_provider.DeploymentMetadata, reason string) (state.DeploymentSnapshot, int, error) { + policy := deploymentRetryPolicy(workload) + var lastErr error + attempts := 0 + + for attempt := 1; attempt <= policy.MaxRetryRollback; attempt++ { + attempts = attempt + d.logger.Info("running rollback attempt", + "phase", logger.PhaseRollback, + "workload", workload.Name, + "attempt", attempt, + "max_attempts", policy.MaxRetryRollback, + ) + + rollbackState, err := d.rollback(ctx, stateMgr, workload, previousState, failedMeta.ManifestDigest, reason) + if err == nil { + return rollbackState, attempts, nil + } + if contextIsDone(err) { + return state.DeploymentSnapshot{}, attempts, err + } + lastErr = err + if attempt == policy.MaxRetryRollback { + break + } + if err := d.sleepBeforeRetry(ctx, policy, attempt, "rollback", workload.Name, err); err != nil { + return state.DeploymentSnapshot{}, attempts, err + } + } + + return state.DeploymentSnapshot{}, attempts, lastErr +} + +func (d *dockerDeployer) sleepBeforeRetry(ctx context.Context, policy config.DeploymentRetry, failedAttempt int, retryPhase, workloadName string, cause error) error { + delay := retryDelay(policy, failedAttempt) + d.logger.Warn("attempt failed; retrying after backoff", + "phase", retryPhase, + "workload", workloadName, + "failed_attempt", failedAttempt, + "next_backoff_ms", delay.Milliseconds(), + "error", cause, + ) + + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return fmt.Errorf("%s retry backoff canceled: %w", retryPhase, ctx.Err()) + case <-timer.C: + return nil + } +} + +func retryDelay(policy config.DeploymentRetry, failedAttempt int) time.Duration { + delay := policy.InitialInterval.Duration + for i := 1; i < failedAttempt; i++ { + next := float64(delay) * policy.Multiplier + if next >= float64(policy.MaxInterval.Duration) { + delay = policy.MaxInterval.Duration + break + } + delay = time.Duration(next) + if delay <= 0 { + delay = policy.MaxInterval.Duration + break + } + } + delay = min(delay, policy.MaxInterval.Duration) + if delay <= 0 { + return 0 + } + + jitterRange := int64(delay / 2) + if jitterRange <= 0 { + return delay + } + jitter := time.Duration(rand.Int63n(jitterRange)) + return min(delay-delay/4+jitter, policy.MaxInterval.Duration) +} + +func (d *dockerDeployer) commitTerminalFailure(stateMgr state.Manager, workload config.Workload, meta git_provider.DeploymentMetadata, deployErr, rollbackErr error, rollbackAttempts int) error { + policy := deploymentRetryPolicy(workload) + reason := fmt.Sprintf("deployment failed: %v", deployErr) + if rollbackErr == nil { + reason += "; rollback succeeded" + } else { + reason += fmt.Sprintf("; rollback failed: %v", rollbackErr) + } + failure := state.TerminalFailureState{ + ManifestDigest: meta.ManifestDigest, + GitSHA: meta.GitSHA, + ForwardDeploymentAttempts: policy.MaxRetryForwardDeployment, + RollbackAttempts: rollbackAttempts, + Reason: reason, + FailedAt: time.Now().UTC(), + } + return stateMgr.CommitTerminalFailure(workload.Name, failure) +} + +func (d *dockerDeployer) commitTerminalFailureOnNoPreviousStateAvailable(stateMgr state.Manager, workload config.Workload, meta git_provider.DeploymentMetadata, deployErr error) error { + policy := deploymentRetryPolicy(workload) + failure := state.TerminalFailureState{ + ManifestDigest: meta.ManifestDigest, + GitSHA: meta.GitSHA, + ForwardDeploymentAttempts: policy.MaxRetryForwardDeployment, + RollbackAttempts: 0, + Reason: fmt.Sprintf("deployment failed: %v, no previous state available, no rollback attempted", deployErr), + FailedAt: time.Now().UTC(), + } + return stateMgr.CommitTerminalFailure(workload.Name, failure) +} + +func deploymentRetryPolicy(workload config.Workload) config.DeploymentRetry { + policy := workload.DeploymentRetry + if policy.MaxRetryForwardDeployment < 1 { + policy.MaxRetryForwardDeployment = 1 + } + if policy.MaxRetryRollback < 1 { + policy.MaxRetryRollback = 1 + } + if policy.InitialInterval.Duration <= 0 { + policy.InitialInterval.Duration = time.Second + } + if policy.MaxInterval.Duration <= 0 { + policy.MaxInterval.Duration = 30 * time.Second + } + if policy.MaxInterval.Duration < policy.InitialInterval.Duration { + policy.MaxInterval.Duration = policy.InitialInterval.Duration + } + if policy.Multiplier < 1 || policy.Multiplier != policy.Multiplier { + policy.Multiplier = 2 + } + return policy +} + +func (d *dockerDeployer) deploy(ctx context.Context, workload config.Workload, meta git_provider.DeploymentMetadata, stateMgr state.Manager) (state.DeploymentSnapshot, error) { + var newState state.DeploymentSnapshot if err := d.runStrategy(ctx, workload, meta); err != nil { + if ctx.Err() != nil { + return newState, ctx.Err() + } return newState, errors.Join(errDeploymentStrategyFailed, err) } d.logger.Info("running deploy command", "phase", logger.PhaseRestart, "workload", workload.Name) if err := d.runDeployCommand(ctx, workload, meta); err != nil { + if ctx.Err() != nil { + return newState, ctx.Err() + } return newState, errors.Join(errDeployCommandFailed, err) } if err := d.waitHealthy(ctx, workload); err != nil { + if ctx.Err() != nil { + return newState, ctx.Err() + } return newState, errors.Join(errHealthCheckFailed, err) } - newState = state.DeploymentState{ - WorkloadName: workload.Name, + newState = state.DeploymentSnapshot{ Image: meta.Image, ImageTag: meta.ImageTag, ManifestDigest: meta.ManifestDigest, @@ -211,6 +403,7 @@ func (d *dockerDeployer) deploy(ctx context.Context, workload config.Workload, m } if err := stateMgr.CommitDeployed(workload.Name, newState); err != nil { d.logger.Error("failed to commit deployment state", "phase", logger.PhaseCommit, "error", err) + return state.DeploymentSnapshot{}, fmt.Errorf("commit deployment state: %w", err) } d.logger.Info("deployment successful", @@ -222,8 +415,8 @@ func (d *dockerDeployer) deploy(ctx context.Context, workload config.Workload, m return newState, nil } -func (d *dockerDeployer) rollback(ctx context.Context, stateMgr state.Manager, workload config.Workload, previousState state.DeploymentState, failedDigest, reason string) (state.DeploymentState, error) { - var rollbackState state.DeploymentState +func (d *dockerDeployer) rollback(ctx context.Context, stateMgr state.Manager, workload config.Workload, previousState state.DeploymentSnapshot, failedDigest, reason string) (state.DeploymentSnapshot, error) { + var rollbackState state.DeploymentSnapshot d.logger.Info("initiating rollback", "phase", logger.PhaseRollback, "workload", workload.Name, @@ -240,19 +433,27 @@ func (d *dockerDeployer) rollback(ctx context.Context, stateMgr state.Manager, w } if err := d.runStrategy(ctx, workload, previousStateMeta); err != nil { + if ctx.Err() != nil { + return rollbackState, ctx.Err() + } return rollbackState, fmt.Errorf("rollback strategy failed: %w (original reason: %s)", err, reason) } if err := d.runDeployCommand(ctx, workload, previousStateMeta); err != nil { + if ctx.Err() != nil { + return rollbackState, ctx.Err() + } d.logger.Error("rollback deploy command failed", "phase", logger.PhaseRollback, "error", err) return rollbackState, fmt.Errorf("rollback deploy command failed: %w (original reason: %s)", err, reason) } if err := d.waitHealthy(ctx, workload); err != nil { + if ctx.Err() != nil { + return rollbackState, ctx.Err() + } d.logger.Error("rollback health check failed", "phase", logger.PhaseRollback, "error", err) return rollbackState, fmt.Errorf("rollback health check failed: %w (original reason: %s)", err, reason) } - rollbackState = state.DeploymentState{ - WorkloadName: workload.Name, + rollbackState = state.DeploymentSnapshot{ Image: previousState.Image, ImageTag: previousState.ImageTag, ManifestDigest: previousState.ManifestDigest, @@ -262,6 +463,7 @@ func (d *dockerDeployer) rollback(ctx context.Context, stateMgr state.Manager, w } if err := stateMgr.CommitRollback(workload.Name, rollbackState); err != nil { d.logger.Error("failed to commit rollback state", "phase", logger.PhaseRollback, "error", err) + return state.DeploymentSnapshot{}, fmt.Errorf("commit rollback state: %w", err) } return rollbackState, nil } @@ -306,6 +508,9 @@ func (d *dockerDeployer) runDeployCommand(ctx context.Context, workload config.W err := cmd.Run() duration := time.Since(start) if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } d.logger.Error("deploy command failed", "phase", logger.PhaseRestart, "workload", workload.Name, @@ -394,3 +599,7 @@ func errorShouldTriggerRollback(err error) bool { } return false } + +func contextIsDone(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} diff --git a/deployment/docker_deployer_test.go b/deployment/docker_deployer_test.go index 5ab7933..8bfdcb1 100644 --- a/deployment/docker_deployer_test.go +++ b/deployment/docker_deployer_test.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "errors" "fmt" "io" "log/slog" @@ -19,33 +20,74 @@ import ( type fakeStateManager struct { loadDeployedErr error - deployed state.DeploymentState - committed state.DeploymentState + deployed state.DeploymentSnapshot + committed state.DeploymentSnapshot committedName string - rollback state.DeploymentState + commitErr error + rollback state.DeploymentSnapshot rollbackName string + rollbackErr error + rollbackCommits int + terminalFailure state.TerminalFailureState + terminalCommits int } -func (m *fakeStateManager) LoadDeployed(string) (state.DeploymentState, error) { +func (m *fakeStateManager) LoadDeployed(string) (state.DeploymentSnapshot, error) { return m.deployed, m.loadDeployedErr } -func (m *fakeStateManager) LoadPrevious(string) (state.DeploymentState, error) { - return state.DeploymentState{}, nil +func (m *fakeStateManager) LoadPrevious(string) (state.DeploymentSnapshot, error) { + return state.DeploymentSnapshot{}, nil } -func (m *fakeStateManager) CommitDeployed(deploymentName string, s state.DeploymentState) error { +func (m *fakeStateManager) LoadTerminalFailure(string) (state.TerminalFailureState, error) { + return state.TerminalFailureState{}, state.ErrDeploymentStateNotFound +} + +func (m *fakeStateManager) CommitDeployed(deploymentName string, s state.DeploymentSnapshot) error { m.committedName = deploymentName m.committed = s - return nil + return m.commitErr } -func (m *fakeStateManager) CommitRollback(deploymentName string, s state.DeploymentState) error { +func (m *fakeStateManager) CommitRollback(deploymentName string, s state.DeploymentSnapshot) error { m.rollbackName = deploymentName m.rollback = s + m.rollbackCommits++ + return m.rollbackErr +} + +func (m *fakeStateManager) CommitTerminalFailure(_ string, s state.TerminalFailureState) error { + m.terminalFailure = s + m.terminalCommits++ return nil } +func (m *fakeStateManager) ClearTerminalFailure(string) error { + m.terminalFailure = state.TerminalFailureState{} + return nil +} + +type fakeNotifier struct { + started int + failed int + success int + lastFail error +} + +func (n *fakeNotifier) NotifyOnNewDeploymentStarted(string, git_provider.DeploymentMetadata) { + n.started++ +} + +func (n *fakeNotifier) NotifyOnDeploymentFailed(_ string, _ git_provider.DeploymentMetadata, err error) { + n.failed++ + n.lastFail = err +} + +func (n *fakeNotifier) NotifyOnDeploymentSuccess(string, state.DeploymentSnapshot) { + n.success++ +} + func TestDockerDeployerDeployCommitsDeploymentStateForWorkload(t *testing.T) { t.Parallel() @@ -67,7 +109,6 @@ func TestDockerDeployerDeployCommitsDeploymentStateForWorkload(t *testing.T) { require.NoError(t, err) require.Equal(t, "api", stateMgr.committedName) - require.Equal(t, "api", stateMgr.committed.WorkloadName) require.Equal(t, meta.ManifestDigest, stateMgr.committed.ManifestDigest) require.Equal(t, newState, stateMgr.committed) } @@ -83,8 +124,10 @@ func TestDockerDeployerDeployTreatsMissingFileAsNoPreviousState(t *testing.T) { } err := testDockerDeployer(workload).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) - require.ErrorContains(t, err, "rollback failed: no previous deployment state") + require.ErrorContains(t, err, "no previous deployment state") require.Empty(t, stateMgr.rollbackName) + require.Equal(t, testDeploymentMetadata().ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Equal(t, 0, stateMgr.terminalFailure.RollbackAttempts) } func TestDockerDeployerDeployTreatsMissingDeploymentAsNoPreviousState(t *testing.T) { @@ -98,16 +141,38 @@ func TestDockerDeployerDeployTreatsMissingDeploymentAsNoPreviousState(t *testing } err := testDockerDeployer(workload).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) - require.ErrorContains(t, err, "rollback failed: no previous deployment state") + require.ErrorContains(t, err, "no previous deployment state") require.Empty(t, stateMgr.rollbackName) + require.Equal(t, testDeploymentMetadata().ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Equal(t, 0, stateMgr.terminalFailure.RollbackAttempts) +} + +func TestDockerDeployerDoesNotStartDeployWhenPreviousStateLoadFailsUnexpectedly(t *testing.T) { + t.Parallel() + + stateMgr := &fakeStateManager{loadDeployedErr: errors.New("permission denied")} + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: "true", + } + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) + require.ErrorContains(t, err, "could not load previous state") + require.Empty(t, stateMgr.committedName) + require.Empty(t, stateMgr.rollbackName) + require.Zero(t, stateMgr.terminalCommits) + require.Zero(t, notifier.started) + require.Zero(t, notifier.failed) + require.Zero(t, notifier.success) } func TestDockerDeployerDeployRollsBackToPreviousState(t *testing.T) { t.Parallel() deployCommand := firstCallFailsThenSucceedsCommand(t) - previousState := state.DeploymentState{ - WorkloadName: "api", + previousState := state.DeploymentSnapshot{ Image: "ghcr.io/example/api", ImageTag: "v0.9.0", ManifestDigest: "sha256:previous", @@ -123,18 +188,287 @@ func TestDockerDeployerDeployRollsBackToPreviousState(t *testing.T) { meta := testDeploymentMetadata() err := testDockerDeployer(workload).Deploy(context.Background(), workload, meta, stateMgr) - require.NoError(t, err) + require.ErrorContains(t, err, "deployment failed") + require.ErrorContains(t, err, "rollback succeeded") require.Equal(t, "api", stateMgr.rollbackName) - require.Equal(t, "api", stateMgr.rollback.WorkloadName) require.Equal(t, previousState.ManifestDigest, stateMgr.rollback.ManifestDigest) require.Equal(t, meta.ManifestDigest, stateMgr.rollback.RollbackFrom) + require.Equal(t, meta.ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Equal(t, 1, stateMgr.terminalFailure.RollbackAttempts) +} + +func TestDockerDeployerPersistsTerminalFailureWhenDeploymentFailsAndRollbackSucceeds(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{deployed: previousState} + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: firstCallFailsThenSucceedsCommand(t), + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 1, + MaxRetryRollback: 3, + InitialInterval: config.Duration{Duration: time.Nanosecond}, + MaxInterval: config.Duration{Duration: time.Nanosecond}, + Multiplier: 1, + }, + } + meta := testDeploymentMetadata() + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(context.Background(), workload, meta, stateMgr) + require.ErrorContains(t, err, "rollback succeeded") + require.Equal(t, meta.ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Equal(t, 1, stateMgr.terminalFailure.ForwardDeploymentAttempts) + require.Equal(t, 1, stateMgr.terminalFailure.RollbackAttempts) + require.Equal(t, 1, stateMgr.terminalCommits) + require.Equal(t, 1, notifier.success) + require.Equal(t, 1, notifier.failed) +} + +func TestDockerDeployerRetriesForwardDeploymentBeforeRollback(t *testing.T) { + t.Parallel() + + deployCommand := firstCallFailsThenSucceedsCommand(t) + stateMgr := &fakeStateManager{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: deployCommand, + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 2, + MaxRetryRollback: 1, + InitialInterval: config.Duration{Duration: time.Nanosecond}, + MaxInterval: config.Duration{Duration: time.Nanosecond}, + Multiplier: 1, + }, + } + + err := testDockerDeployer(workload).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) + require.NoError(t, err) + require.Equal(t, "api", stateMgr.committedName) + require.Empty(t, stateMgr.rollbackName) +} + +func TestDockerDeployerPersistsTerminalFailureWhenDeploymentAndRollbackRetriesExhaust(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{deployed: previousState} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: "false", + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 2, + MaxRetryRollback: 2, + InitialInterval: config.Duration{Duration: time.Nanosecond}, + MaxInterval: config.Duration{Duration: time.Nanosecond}, + Multiplier: 1, + }, + } + meta := testDeploymentMetadata() + + err := testDockerDeployer(workload).Deploy(context.Background(), workload, meta, stateMgr) + require.ErrorContains(t, err, "deployment and rollback failed") + require.Equal(t, meta.ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Equal(t, 2, stateMgr.terminalFailure.ForwardDeploymentAttempts) + require.Equal(t, 2, stateMgr.terminalFailure.RollbackAttempts) +} + +func TestDockerDeployerDoesNotRollbackOrPersistTerminalFailureWhenContextCanceled(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{deployed: previousState} + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: "sleep 10", + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 2, + MaxRetryRollback: 2, + InitialInterval: config.Duration{Duration: time.Nanosecond}, + MaxInterval: config.Duration{Duration: time.Nanosecond}, + Multiplier: 1, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(ctx, workload, testDeploymentMetadata(), stateMgr) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, stateMgr.rollbackName) + require.Zero(t, stateMgr.terminalCommits) + require.Equal(t, 1, notifier.started) + require.Zero(t, notifier.failed) + require.Zero(t, notifier.success) +} + +func TestDockerDeployerDoesNotRollbackOrPersistTerminalFailureWhenBackoffCanceled(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{deployed: previousState} + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: "false", + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 2, + MaxRetryRollback: 2, + InitialInterval: config.Duration{Duration: time.Hour}, + MaxInterval: config.Duration{Duration: time.Hour}, + Multiplier: 1, + }, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(ctx, workload, testDeploymentMetadata(), stateMgr) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Empty(t, stateMgr.rollbackName) + require.Zero(t, stateMgr.terminalCommits) + require.Equal(t, 1, notifier.started) + require.Zero(t, notifier.failed) + require.Zero(t, notifier.success) +} + +func TestDockerDeployerCommitDeployedFailureReturnsErrorWithoutSuccessNotification(t *testing.T) { + t.Parallel() + + stateMgr := &fakeStateManager{commitErr: errors.New("disk full")} + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: "true", + } + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) + require.ErrorContains(t, err, "commit deployment state") + require.Equal(t, "api", stateMgr.committedName) + require.Zero(t, notifier.success) + require.Equal(t, 1, notifier.failed) + require.Zero(t, stateMgr.terminalCommits) +} + +func TestDockerDeployerCommitRollbackFailureReturnsErrorAndPersistsTerminalFailure(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{ + deployed: previousState, + rollbackErr: errors.New("state write failed"), + } + notifier := &fakeNotifier{} + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: firstCallFailsThenSucceedsCommand(t), + } + meta := testDeploymentMetadata() + + err := testDockerDeployerWithNotifier(workload, notifier).Deploy(context.Background(), workload, meta, stateMgr) + require.ErrorContains(t, err, "deployment and rollback failed") + require.ErrorContains(t, err, "commit rollback state") + require.Equal(t, "api", stateMgr.rollbackName) + require.Equal(t, meta.ManifestDigest, stateMgr.terminalFailure.ManifestDigest) + require.Zero(t, notifier.success) + require.Equal(t, 2, notifier.failed) +} + +func TestDockerDeployerRetriesRollbackAttempts(t *testing.T) { + t.Parallel() + + previousState := state.DeploymentSnapshot{ + Image: "ghcr.io/example/api", + ImageTag: "v0.9.0", + ManifestDigest: "sha256:previous", + GitSHA: "previous-sha", + DeployedAt: time.Now().UTC(), + } + stateMgr := &fakeStateManager{ + deployed: previousState, + rollbackErr: errors.New("state write failed"), + } + workload := config.Workload{ + Name: "api", + ContainerName: "api", + DeployCommand: firstCallFailsThenSucceedsCommand(t), + DeploymentRetry: config.DeploymentRetry{ + MaxRetryForwardDeployment: 1, + MaxRetryRollback: 3, + InitialInterval: config.Duration{Duration: time.Nanosecond}, + MaxInterval: config.Duration{Duration: time.Nanosecond}, + Multiplier: 1, + }, + } + + err := testDockerDeployer(workload).Deploy(context.Background(), workload, testDeploymentMetadata(), stateMgr) + require.ErrorContains(t, err, "deployment and rollback failed") + require.Equal(t, 3, stateMgr.rollbackCommits) + require.Equal(t, 3, stateMgr.terminalFailure.RollbackAttempts) +} + +func TestRetryDelayClampsLargeMultiplierAndJitterToMaxInterval(t *testing.T) { + t.Parallel() + + policy := config.DeploymentRetry{ + InitialInterval: config.Duration{Duration: time.Second}, + MaxInterval: config.Duration{Duration: 10 * time.Second}, + Multiplier: 1e308, + } + + for range 100 { + delay := retryDelay(policy, 2) + require.Positive(t, delay) + require.LessOrEqual(t, delay, policy.MaxInterval.Duration) + } } func testDockerDeployer(workload config.Workload) *dockerDeployer { + return testDockerDeployerWithNotifier(workload, notifier.Noop{}) +} + +func testDockerDeployerWithNotifier(workload config.Workload, n notifier.Notifier) *dockerDeployer { return NewDockerDeployer( workload, - notifier.Noop{}, + n, slog.New(slog.NewTextHandler(io.Discard, nil)), ).(*dockerDeployer) } diff --git a/notifier/notifier.go b/notifier/notifier.go index a4b2c19..fbb2d92 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -13,7 +13,7 @@ import ( type Notifier interface { NotifyOnNewDeploymentStarted(workloadName string, meta git_provider.DeploymentMetadata) NotifyOnDeploymentFailed(workloadName string, meta git_provider.DeploymentMetadata, err error) - NotifyOnDeploymentSuccess(dep state.DeploymentState) + NotifyOnDeploymentSuccess(workloadName string, dep state.DeploymentSnapshot) } type Noop struct{} @@ -21,4 +21,4 @@ type Noop struct{} func (Noop) NotifyOnNewDeploymentStarted(workloadName string, meta git_provider.DeploymentMetadata) {} func (Noop) NotifyOnDeploymentFailed(workloadName string, meta git_provider.DeploymentMetadata, err error) { } -func (Noop) NotifyOnDeploymentSuccess(dep state.DeploymentState) {} +func (Noop) NotifyOnDeploymentSuccess(workloadName string, dep state.DeploymentSnapshot) {} diff --git a/notifier/slack_notifier.go b/notifier/slack_notifier.go index aa3eafe..0a5d7d8 100644 --- a/notifier/slack_notifier.go +++ b/notifier/slack_notifier.go @@ -65,8 +65,8 @@ func (sl *Slack) NotifyOnDeploymentFailed(workloadName string, meta git_provider } } -func (sl *Slack) NotifyOnDeploymentSuccess(dep state.DeploymentState) { - msg := slackNotificationFromDeploymentState(dep, deploymentStateSuccess) +func (sl *Slack) NotifyOnDeploymentSuccess(workloadName string, dep state.DeploymentSnapshot) { + msg := slackNotificationFromDeploymentState(workloadName, dep, deploymentStateSuccess) err := sl.send(msg) if err != nil { sl.log.Error("Failed to send slack notification", "error", err) @@ -85,10 +85,10 @@ func slackNotificationFromDeploymentMeta(workloadName string, meta git_provider. } } -func slackNotificationFromDeploymentState(depState state.DeploymentState, st deploymentState) slackNotification { +func slackNotificationFromDeploymentState(workloadName string, depState state.DeploymentSnapshot, st deploymentState) slackNotification { return slackNotification{ State: st, - Workload: depState.WorkloadName, + Workload: workloadName, Image: depState.Image, ImageTag: depState.ImageTag, ManifestDigest: depState.ManifestDigest, diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index c9eb04a..10b2cb0 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -25,6 +25,15 @@ type Reconciler struct { log *slog.Logger } +type terminalFailureStatus int + +const ( + terminalFailureUnknown terminalFailureStatus = iota + terminalFailureNotFound + terminalFailureFound + terminalFailureStale +) + func New( workload config.Workload, gitProvider git_provider.GitProvider, @@ -92,6 +101,15 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) { return } + terminalFailure, err := r.terminalFailureStatus(desiredState) + if err != nil { + r.log.Error("failed to load terminal failure state", + "phase", logger.PhaseDrift, + "error", err, + ) + return + } + hasDrift, driftReason, err := r.hasDrift(ctx, currentState, desiredState) if err != nil { r.log.Error("failed to observe runtime state", @@ -101,6 +119,15 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) { return } if !hasDrift { + if terminalFailure != terminalFailureNotFound { + if err = r.stateMgr.ClearTerminalFailure(r.workload.Name); err != nil { + r.log.Error("failed to clear terminal failure for already deployed desired state", + "phase", logger.PhaseCommit, + "error", err, + ) + return + } + } r.log.Debug("no drift detected", "phase", logger.PhaseNoop, "digest", desiredState.ManifestDigest, @@ -109,6 +136,15 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) { return } + if terminalFailure == terminalFailureFound { + r.log.Warn("automatic deployment skipped after retry exhaustion", + "phase", logger.PhaseDrift, + "digest", desiredState.ManifestDigest, + "workload", r.workload.Name, + ) + return + } + r.log.Info("drift detected; deployment required", "phase", logger.PhaseDrift, "drift_reason", driftReason, @@ -122,12 +158,39 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) { return } + if terminalFailure == terminalFailureStale { + if err := r.stateMgr.ClearTerminalFailure(r.workload.Name); err != nil { + r.log.Error("failed to clear stale terminal failure after successful deployment", + "phase", logger.PhaseCommit, + "error", err, + ) + return + } + } + r.log.Info("reconcile cycle completed successfully", "digest", desiredState.ManifestDigest, "duration_ms", time.Since(start).Milliseconds(), ) } +func (r *Reconciler) terminalFailureStatus(desired git_provider.DeploymentMetadata) (terminalFailureStatus, error) { + failure, err := r.stateMgr.LoadTerminalFailure(r.workload.Name) + if err != nil { + if !state.IsNotFound(err) { + return terminalFailureUnknown, fmt.Errorf("failed to load terminal failure status: %w", err) + + } + return terminalFailureNotFound, nil + } + + if failure.ManifestDigest == desired.ManifestDigest { + return terminalFailureFound, nil + } + + return terminalFailureStale, nil +} + func (r *Reconciler) fetchDesiredState(ctx context.Context) (git_provider.DeploymentMetadata, error) { if r.gitProvider == nil { return git_provider.DeploymentMetadata{}, fmt.Errorf("workload %q has no git provider", r.workload.Name) @@ -149,7 +212,7 @@ func (r *Reconciler) validateDesiredState(meta git_provider.DeploymentMetadata) return nil } -func (r *Reconciler) hasDrift(ctx context.Context, actual state.DeploymentState, desired git_provider.DeploymentMetadata) (bool, string, error) { +func (r *Reconciler) hasDrift(ctx context.Context, actual state.DeploymentSnapshot, desired git_provider.DeploymentMetadata) (bool, string, error) { if actual.ManifestDigest == "" { return true, "no current deployment state", nil } diff --git a/reconciler/reconciler_test.go b/reconciler/reconciler_test.go index c60f936..50af38e 100644 --- a/reconciler/reconciler_test.go +++ b/reconciler/reconciler_test.go @@ -42,23 +42,45 @@ func (d *fakeDeployer) Deploy(context.Context, config.Workload, git_provider.Dep } type fakeStateManager struct { - loadDeployedErr error - deployed state.DeploymentState + loadDeployedErr error + deployed state.DeploymentSnapshot + loadTerminalFailure state.TerminalFailureState + loadTerminalErr error + clearFailureCalls int } -func (m fakeStateManager) LoadDeployed(string) (state.DeploymentState, error) { +func (m *fakeStateManager) LoadDeployed(string) (state.DeploymentSnapshot, error) { return m.deployed, m.loadDeployedErr } -func (m fakeStateManager) LoadPrevious(string) (state.DeploymentState, error) { - return state.DeploymentState{}, nil +func (m *fakeStateManager) LoadPrevious(string) (state.DeploymentSnapshot, error) { + return state.DeploymentSnapshot{}, nil } -func (m fakeStateManager) CommitDeployed(string, state.DeploymentState) error { +func (m *fakeStateManager) LoadTerminalFailure(string) (state.TerminalFailureState, error) { + if m.loadTerminalErr != nil { + return state.TerminalFailureState{}, m.loadTerminalErr + } + if m.loadTerminalFailure.ManifestDigest == "" { + return state.TerminalFailureState{}, state.ErrDeploymentStateNotFound + } + return m.loadTerminalFailure, nil +} + +func (m *fakeStateManager) CommitDeployed(string, state.DeploymentSnapshot) error { + return nil +} + +func (m *fakeStateManager) CommitRollback(string, state.DeploymentSnapshot) error { + return nil +} + +func (m *fakeStateManager) CommitTerminalFailure(string, state.TerminalFailureState) error { return nil } -func (m fakeStateManager) CommitRollback(string, state.DeploymentState) error { +func (m *fakeStateManager) ClearTerminalFailure(string) error { + m.clearFailureCalls++ return nil } @@ -95,6 +117,99 @@ func TestReconcileSkipsDeployWhenStateLoadFailsUnexpectedly(t *testing.T) { require.Zero(t, deployer.deployCalls) } +func TestReconcileSkipsDeployWhenTerminalFailureLoadFailsUnexpectedly(t *testing.T) { + t.Parallel() + + deployer := &fakeDeployer{} + r := testReconciler(state.ErrDeploymentStateNotFound, deployer) + r.stateMgr = &fakeStateManager{ + loadDeployedErr: state.ErrDeploymentStateNotFound, + loadTerminalErr: errors.New("permission denied"), + } + + r.reconcileOnce(context.Background()) + + require.Zero(t, deployer.deployCalls) +} + +func TestReconcileSkipsDeployWhenTerminalFailureMatchesDesiredDigest(t *testing.T) { + t.Parallel() + + deployer := &fakeDeployer{} + r := testReconciler(state.ErrDeploymentStateNotFound, deployer) + r.stateMgr = &fakeStateManager{ + loadDeployedErr: state.ErrDeploymentStateNotFound, + loadTerminalFailure: state.TerminalFailureState{ + ManifestDigest: "sha256:desired", + }, + } + + r.reconcileOnce(context.Background()) + + require.Zero(t, deployer.deployCalls) +} + +func TestReconcileDeploysWhenTerminalFailureIsForDifferentDigest(t *testing.T) { + t.Parallel() + + deployer := &fakeDeployer{} + r := testReconciler(state.ErrDeploymentStateNotFound, deployer) + stateMgr := &fakeStateManager{ + loadDeployedErr: state.ErrDeploymentStateNotFound, + loadTerminalFailure: state.TerminalFailureState{ + ManifestDigest: "sha256:old", + }, + } + r.stateMgr = stateMgr + + r.reconcileOnce(context.Background()) + + require.Equal(t, 1, deployer.deployCalls) + require.Equal(t, 1, stateMgr.clearFailureCalls) +} + +func TestReconcileClearsMatchingTerminalFailureWhenDesiredIsAlreadyDeployed(t *testing.T) { + t.Parallel() + + deployer := &fakeDeployer{} + r := testReconciler(nil, deployer) + stateMgr := &fakeStateManager{ + deployed: state.DeploymentSnapshot{ + ManifestDigest: "sha256:desired", + }, + loadTerminalFailure: state.TerminalFailureState{ + ManifestDigest: "sha256:desired", + }, + } + r.stateMgr = stateMgr + + r.reconcileOnce(context.Background()) + + require.Zero(t, deployer.deployCalls) + require.Equal(t, 1, stateMgr.clearFailureCalls) +} + +func TestReconcileClearsStaleTerminalFailureWhenDesiredIsAlreadyDeployed(t *testing.T) { + t.Parallel() + + deployer := &fakeDeployer{} + r := testReconciler(nil, deployer) + stateMgr := &fakeStateManager{ + deployed: state.DeploymentSnapshot{ + ManifestDigest: "sha256:desired", + }, + loadTerminalFailure: state.TerminalFailureState{ + ManifestDigest: "sha256:old", + }, + } + r.stateMgr = stateMgr + + r.reconcileOnce(context.Background()) + + require.Zero(t, deployer.deployCalls) + require.Equal(t, 1, stateMgr.clearFailureCalls) +} + func testReconciler(loadDeployedErr error, deployer *fakeDeployer) *Reconciler { return &Reconciler{ workload: config.Workload{ @@ -108,7 +223,7 @@ func testReconciler(loadDeployedErr error, deployer *fakeDeployer) *Reconciler { ManifestDigest: "sha256:desired", }}, deployer: deployer, - stateMgr: fakeStateManager{loadDeployedErr: loadDeployedErr}, + stateMgr: &fakeStateManager{loadDeployedErr: loadDeployedErr}, log: slog.New(slog.NewTextHandler(io.Discard, nil)), } } diff --git a/state/local_fs.go b/state/local_fs.go index 6db6369..f717b0e 100644 --- a/state/local_fs.go +++ b/state/local_fs.go @@ -7,10 +7,11 @@ import ( "os" "path/filepath" "sync" + "time" ) type deploymentsStateFile struct { - Deployments []DeploymentStateNew `json:"deployments"` + Deployments []DeploymentState `json:"deployments"` } // managerWithLocalFileSystem handles reading and writing deployment state for all workloads. @@ -32,9 +33,9 @@ func NewManagerWithLocalFileSystem(path string) (Manager, error) { } // LoadDeployed returns the currently deployed state for deploymentName. -func (m *managerWithLocalFileSystem) LoadDeployed(deploymentName string) (DeploymentState, error) { +func (m *managerWithLocalFileSystem) LoadDeployed(deploymentName string) (DeploymentSnapshot, error) { if deploymentName == "" { - return DeploymentState{}, fmt.Errorf("deployment name is required") + return DeploymentSnapshot{}, fmt.Errorf("deployment name is required") } m.mu.Lock() @@ -42,18 +43,18 @@ func (m *managerWithLocalFileSystem) LoadDeployed(deploymentName string) (Deploy record, err := m.loadDeployment(deploymentName) if err != nil { - return DeploymentState{}, err + return DeploymentSnapshot{}, err } - if isZeroDeploymentState(record.Current) { - return DeploymentState{}, ErrDeploymentStateNotFound + if isZeroDeploymentSnapshot(record.Current) { + return DeploymentSnapshot{}, ErrDeploymentStateNotFound } return record.Current, nil } // LoadPrevious returns the previous deployment state for deploymentName. -func (m *managerWithLocalFileSystem) LoadPrevious(deploymentName string) (DeploymentState, error) { +func (m *managerWithLocalFileSystem) LoadPrevious(deploymentName string) (DeploymentSnapshot, error) { if deploymentName == "" { - return DeploymentState{}, fmt.Errorf("deployment name is required") + return DeploymentSnapshot{}, fmt.Errorf("deployment name is required") } m.mu.Lock() @@ -61,17 +62,35 @@ func (m *managerWithLocalFileSystem) LoadPrevious(deploymentName string) (Deploy record, err := m.loadDeployment(deploymentName) if err != nil { - return DeploymentState{}, err + return DeploymentSnapshot{}, err } - if isZeroDeploymentState(record.Previous) { - return DeploymentState{}, ErrDeploymentStateNotFound + if isZeroDeploymentSnapshot(record.Previous) { + return DeploymentSnapshot{}, ErrDeploymentStateNotFound } return record.Previous, nil } +func (m *managerWithLocalFileSystem) LoadTerminalFailure(deploymentName string) (TerminalFailureState, error) { + if deploymentName == "" { + return TerminalFailureState{}, fmt.Errorf("deployment name is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + record, err := m.loadDeployment(deploymentName) + if err != nil { + return TerminalFailureState{}, err + } + if record.TerminalFailure == nil || *record.TerminalFailure == (TerminalFailureState{}) { + return TerminalFailureState{}, ErrDeploymentStateNotFound + } + return *record.TerminalFailure, nil +} + // CommitDeployed atomically persists s as the current deployed state for deploymentName. // The current state, if any, is promoted to previous first. -func (m *managerWithLocalFileSystem) CommitDeployed(deploymentName string, s DeploymentState) error { +func (m *managerWithLocalFileSystem) CommitDeployed(deploymentName string, s DeploymentSnapshot) error { if deploymentName == "" { return fmt.Errorf("deployment name is required") } @@ -85,18 +104,20 @@ func (m *managerWithLocalFileSystem) CommitDeployed(deploymentName string, s Dep } record := upsertDeployment(&fileState, deploymentName) - if !isZeroDeploymentState(record.Current) { + if !isZeroDeploymentSnapshot(record.Current) { record.Previous = record.Current } - s.WorkloadName = deploymentName record.Current = s + record.TerminalFailure = nil return m.writeAtomic(fileState) } // CommitRollback persists s as the current deployed state after a rollback. -// It does not update Previous, preserving the last known-good rollback target. -func (m *managerWithLocalFileSystem) CommitRollback(deploymentName string, s DeploymentState) error { +// It does not update Previous or clear TerminalFailure, preserving the last +// known-good rollback target and any terminal marker for the failed desired +// deployment. +func (m *managerWithLocalFileSystem) CommitRollback(deploymentName string, s DeploymentSnapshot) error { if deploymentName == "" { return fmt.Errorf("deployment name is required") } @@ -110,16 +131,59 @@ func (m *managerWithLocalFileSystem) CommitRollback(deploymentName string, s Dep } record := upsertDeployment(&fileState, deploymentName) - s.WorkloadName = deploymentName record.Current = s return m.writeAtomic(fileState) } -func (m *managerWithLocalFileSystem) loadDeployment(deploymentName string) (DeploymentStateNew, error) { +func (m *managerWithLocalFileSystem) CommitTerminalFailure(deploymentName string, s TerminalFailureState) error { + if deploymentName == "" { + return fmt.Errorf("deployment name is required") + } + if s.ManifestDigest == "" { + return fmt.Errorf("terminal failure manifest digest is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + fileState, err := m.loadAll() + if err != nil && !errors.Is(err, ErrFileNotFound) { + return err + } + + record := upsertDeployment(&fileState, deploymentName) + if s.FailedAt.IsZero() { + s.FailedAt = time.Now().UTC() + } + record.TerminalFailure = &s + + return m.writeAtomic(fileState) +} + +func (m *managerWithLocalFileSystem) ClearTerminalFailure(deploymentName string) error { + if deploymentName == "" { + return fmt.Errorf("deployment name is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + fileState, err := m.loadAll() + if err != nil { + return err + } + + record := upsertDeployment(&fileState, deploymentName) + record.TerminalFailure = nil + + return m.writeAtomic(fileState) +} + +func (m *managerWithLocalFileSystem) loadDeployment(deploymentName string) (DeploymentState, error) { fileState, err := m.loadAll() if err != nil { - return DeploymentStateNew{}, err + return DeploymentState{}, err } for _, deployment := range fileState.Deployments { @@ -127,7 +191,7 @@ func (m *managerWithLocalFileSystem) loadDeployment(deploymentName string) (Depl return deployment, nil } } - return DeploymentStateNew{}, ErrDeploymentStateNotFound + return DeploymentState{}, ErrDeploymentStateNotFound } func (m *managerWithLocalFileSystem) loadAll() (deploymentsStateFile, error) { @@ -171,19 +235,19 @@ func (m *managerWithLocalFileSystem) writeAtomic(fileState deploymentsStateFile) return nil } -func upsertDeployment(fileState *deploymentsStateFile, deploymentName string) *DeploymentStateNew { +func upsertDeployment(fileState *deploymentsStateFile, deploymentName string) *DeploymentState { for i := range fileState.Deployments { if fileState.Deployments[i].WorkloadName == deploymentName { return &fileState.Deployments[i] } } - fileState.Deployments = append(fileState.Deployments, DeploymentStateNew{ + fileState.Deployments = append(fileState.Deployments, DeploymentState{ WorkloadName: deploymentName, }) return &fileState.Deployments[len(fileState.Deployments)-1] } -func isZeroDeploymentState(s DeploymentState) bool { - return s == DeploymentState{} +func isZeroDeploymentSnapshot(s DeploymentSnapshot) bool { + return s == DeploymentSnapshot{} } diff --git a/state/local_fs_test.go b/state/local_fs_test.go index efec139..23b7280 100644 --- a/state/local_fs_test.go +++ b/state/local_fs_test.go @@ -17,7 +17,7 @@ func TestLocalFileSystemManagerCommitDeployedWritesFirstDeployment(t *testing.T) manager, err := NewManagerWithLocalFileSystem(stateFile) require.NoError(t, err) - expected := DeploymentState{ + expected := DeploymentSnapshot{ Image: "ghcr.io/example/app:v1.0.0", ManifestDigest: "v1.0.0", GitSHA: "sha-v1.0.0", @@ -28,7 +28,6 @@ func TestLocalFileSystemManagerCommitDeployedWritesFirstDeployment(t *testing.T) actual, err := manager.LoadDeployed("api") require.NoError(t, err) - require.Equal(t, "api", actual.WorkloadName) require.Equal(t, expected.Image, actual.Image) require.Equal(t, expected.ManifestDigest, actual.ManifestDigest) require.Equal(t, expected.GitSHA, actual.GitSHA) @@ -44,13 +43,13 @@ func TestLocalFileSystemManagerCommitDeployedPromotesPreviousDeployment(t *testi manager, err := NewManagerWithLocalFileSystem(stateFile) require.NoError(t, err) - first := DeploymentState{ + first := DeploymentSnapshot{ Image: "ghcr.io/example/app:v1.0.0", ManifestDigest: "v1.0.0", GitSHA: "sha-v1.0.0", DeployedAt: time.Now().UTC(), } - second := DeploymentState{ + second := DeploymentSnapshot{ Image: "ghcr.io/example/app:v1.1.0", ManifestDigest: "v1.1.0", GitSHA: "sha-v1.1.0", @@ -76,13 +75,13 @@ func TestLocalFileSystemManagerStoresMultipleDeploymentsInOneFile(t *testing.T) manager, err := NewManagerWithLocalFileSystem(stateFile) require.NoError(t, err) - api := DeploymentState{ + api := DeploymentSnapshot{ Image: "ghcr.io/example/api:v1.0.0", ManifestDigest: "api-v1.0.0", GitSHA: "api-sha", DeployedAt: time.Now().UTC(), } - worker := DeploymentState{ + worker := DeploymentSnapshot{ Image: "ghcr.io/example/worker:v1.0.0", ManifestDigest: "worker-v1.0.0", GitSHA: "worker-sha", @@ -119,19 +118,19 @@ func TestLocalFileSystemManagerCommitRollbackPreservesPreviousDeployment(t *test manager, err := NewManagerWithLocalFileSystem(stateFile) require.NoError(t, err) - first := DeploymentState{ + first := DeploymentSnapshot{ Image: "ghcr.io/example/api:v1.0.0", ManifestDigest: "api-v1.0.0", GitSHA: "api-sha-v1", DeployedAt: time.Now().UTC(), } - second := DeploymentState{ + second := DeploymentSnapshot{ Image: "ghcr.io/example/api:v1.1.0", ManifestDigest: "api-v1.1.0", GitSHA: "api-sha-v2", DeployedAt: time.Now().UTC(), } - rollback := DeploymentState{ + rollback := DeploymentSnapshot{ Image: first.Image, ManifestDigest: first.ManifestDigest, GitSHA: first.GitSHA, @@ -153,6 +152,111 @@ func TestLocalFileSystemManagerCommitRollbackPreservesPreviousDeployment(t *test require.Equal(t, first.ManifestDigest, previous.ManifestDigest) } +func TestLocalFileSystemManagerCommitRollbackPreservesTerminalFailure(t *testing.T) { + t.Parallel() + + stateFile := filepath.Join(t.TempDir(), "deployments_state.json") + manager, err := NewManagerWithLocalFileSystem(stateFile) + require.NoError(t, err) + + first := DeploymentSnapshot{ + Image: "ghcr.io/example/api:v1.0.0", + ManifestDigest: "api-v1.0.0", + GitSHA: "api-sha-v1", + DeployedAt: time.Now().UTC(), + } + failure := TerminalFailureState{ + ManifestDigest: "api-v1.1.0", + ForwardDeploymentAttempts: 3, + RollbackAttempts: 1, + Reason: "deployment failed; rollback succeeded", + } + + require.NoError(t, manager.CommitDeployed("api", first)) + require.NoError(t, manager.CommitTerminalFailure("api", failure)) + require.NoError(t, manager.CommitRollback("api", DeploymentSnapshot{ + Image: first.Image, + ManifestDigest: first.ManifestDigest, + GitSHA: first.GitSHA, + DeployedAt: time.Now().UTC(), + RollbackFrom: failure.ManifestDigest, + })) + + actual, err := manager.LoadTerminalFailure("api") + require.NoError(t, err) + require.Equal(t, failure.ManifestDigest, actual.ManifestDigest) + require.Equal(t, 1, actual.RollbackAttempts) +} + +func TestLocalFileSystemManagerTerminalFailureLifecycle(t *testing.T) { + t.Parallel() + + stateFile := filepath.Join(t.TempDir(), "deployments_state.json") + manager, err := NewManagerWithLocalFileSystem(stateFile) + require.NoError(t, err) + + failure := TerminalFailureState{ + ManifestDigest: "sha256:failed", + GitSHA: "failed-sha", + ForwardDeploymentAttempts: 3, + RollbackAttempts: 2, + Reason: "rollback failed", + } + + require.NoError(t, manager.CommitTerminalFailure("api", failure)) + + actual, err := manager.LoadTerminalFailure("api") + require.NoError(t, err) + require.Equal(t, failure.ManifestDigest, actual.ManifestDigest) + require.Equal(t, 3, actual.ForwardDeploymentAttempts) + require.False(t, actual.FailedAt.IsZero()) + + require.NoError(t, manager.ClearTerminalFailure("api")) + _, err = manager.LoadTerminalFailure("api") + require.ErrorIs(t, err, ErrDeploymentStateNotFound) +} + +func TestLocalFileSystemManagerOmitsAbsentTerminalFailure(t *testing.T) { + t.Parallel() + + stateFile := filepath.Join(t.TempDir(), "deployments_state.json") + manager, err := NewManagerWithLocalFileSystem(stateFile) + require.NoError(t, err) + + require.NoError(t, manager.CommitDeployed("api", DeploymentSnapshot{ + Image: "ghcr.io/example/api:v1.0.0", + ManifestDigest: "sha256:ok", + GitSHA: "ok-sha", + DeployedAt: time.Now().UTC(), + })) + + raw, err := os.ReadFile(stateFile) + require.NoError(t, err) + require.NotContains(t, string(raw), "terminal_failure") +} + +func TestLocalFileSystemManagerCommitDeployedClearsTerminalFailure(t *testing.T) { + t.Parallel() + + stateFile := filepath.Join(t.TempDir(), "deployments_state.json") + manager, err := NewManagerWithLocalFileSystem(stateFile) + require.NoError(t, err) + + require.NoError(t, manager.CommitTerminalFailure("api", TerminalFailureState{ + ManifestDigest: "sha256:failed", + Reason: "failed", + })) + require.NoError(t, manager.CommitDeployed("api", DeploymentSnapshot{ + Image: "ghcr.io/example/api:v1.0.0", + ManifestDigest: "sha256:ok", + GitSHA: "ok-sha", + DeployedAt: time.Now().UTC(), + })) + + _, err = manager.LoadTerminalFailure("api") + require.ErrorIs(t, err, ErrDeploymentStateNotFound) +} + func TestLocalFileSystemManagerLoadDeployedMissingFile(t *testing.T) { t.Parallel() @@ -172,7 +276,7 @@ func TestLocalFileSystemManagerLoadDeployedMissingDeployment(t *testing.T) { manager, err := NewManagerWithLocalFileSystem(stateFile) require.NoError(t, err) - require.NoError(t, manager.CommitDeployed("api", DeploymentState{ + require.NoError(t, manager.CommitDeployed("api", DeploymentSnapshot{ Image: "ghcr.io/example/api:v1.0.0", ManifestDigest: "api-v1.0.0", GitSHA: "api-sha", diff --git a/state/state.go b/state/state.go index c007c9a..7927e75 100644 --- a/state/state.go +++ b/state/state.go @@ -16,10 +16,13 @@ import ( ) type Manager interface { - LoadDeployed(deploymentName string) (DeploymentState, error) - LoadPrevious(deploymentName string) (DeploymentState, error) - CommitDeployed(deploymentName string, s DeploymentState) error - CommitRollback(deploymentName string, s DeploymentState) error + LoadDeployed(deploymentName string) (DeploymentSnapshot, error) + LoadPrevious(deploymentName string) (DeploymentSnapshot, error) + LoadTerminalFailure(deploymentName string) (TerminalFailureState, error) + CommitDeployed(deploymentName string, s DeploymentSnapshot) error + CommitRollback(deploymentName string, s DeploymentSnapshot) error + CommitTerminalFailure(deploymentName string, s TerminalFailureState) error + ClearTerminalFailure(deploymentName string) error } var ( @@ -31,17 +34,16 @@ func IsNotFound(err error) bool { return errors.Is(err, ErrFileNotFound) || errors.Is(err, ErrDeploymentStateNotFound) } -// DeploymentStateNew represents the current and previous deployment for one workload. -type DeploymentStateNew struct { - WorkloadName string `json:"workload_name" yaml:"workload_name"` - Current DeploymentState `json:"current" yaml:"current"` - Previous DeploymentState `json:"previous" yaml:"previous"` -} - -// DeploymentState represents a point-in-time snapshot of what is deployed. +// DeploymentState represents the current and previous deployment for one workload. type DeploymentState struct { - WorkloadName string `json:"workload_name"` + WorkloadName string `json:"workload_name" yaml:"workload_name"` + Current DeploymentSnapshot `json:"current" yaml:"current"` + Previous DeploymentSnapshot `json:"previous" yaml:"previous"` + TerminalFailure *TerminalFailureState `json:"terminal_failure,omitempty" yaml:"terminal_failure,omitempty"` +} +// DeploymentSnapshot represents a point-in-time snapshot of what is deployed. +type DeploymentSnapshot struct { // Image is the full image reference that is deployed. Image string `json:"image"` @@ -62,3 +64,14 @@ type DeploymentState struct { // It contains the digest that failed, for audit purposes. RollbackFrom string `json:"rollback_from,omitempty"` } + +// TerminalFailureState blocks automatic retries for a desired deployment that +// exhausted its forward deployment retry budget. +type TerminalFailureState struct { + ManifestDigest string `json:"manifest_digest"` + GitSHA string `json:"git_sha,omitempty"` + ForwardDeploymentAttempts int `json:"forward_deployment_attempts"` + RollbackAttempts int `json:"rollback_attempts,omitempty"` + Reason string `json:"reason"` + FailedAt time.Time `json:"failed_at"` +}