bounded forward deployment and rollback deployment attempts

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## v0.4.2 - 2026-10-12
+
+### Added
+
+- Added `deployment_retry` configuration for bounded forward deployment and rollback attempts.
+- Added top-level `deployment_retry` defaults that apply to every workload.
+- Added workload-level `deployment_retry` overrides; workload settings override only the fields they explicitly set.
+- Added exponential retry backoff with jitter for deployment strategy, deploy command, health-check, and rollback failures.
+- Added persisted terminal failure state for deployment digests that exhaust forward retry attempts, preventing repeated automatic deployment loops.
+
+### Changed
+
+- Forward deployment attempts default to `1`, preserving previous single-attempt behavior unless retry settings are configured.
+- Rollback attempts default to `1`, with retry delay defaults of `1s` initial interval, `30s` max interval, and multiplier `2`.
+- Rollback now starts only after forward deployment attempts are exhausted.
+- Reconciliation now skips automatic deployment when a terminal failure matches the desired manifest digest.
+- Reconciliation allows deployment of a newer desired manifest digest when an existing terminal failure belongs to an older digest.
+- Successful forward deployment commits now clear terminal failure state for the workload.
+- Successful rollback after forward retry exhaustion now persists terminal failure for the failed desired digest, preventing the next reconciliation cycle from retrying the same known-bad digest automatically.
+- Deployment state types were clarified: `DeploymentState` is the outer per-workload record, while `DeploymentSnapshot` is the current or previous deployed snapshot.
+- Slack success notifications now receive workload name separately from the deployment snapshot.
+
+### Fixed
+
+- Context cancellation during deployment retry backoff and rollback retry backoff now stops retries promptly.
+- Unexpected previous-state load errors now abort deployment instead of attempting rollback with an empty previous snapshot.
+- Unexpected terminal-failure load errors now abort reconciliation instead of treating terminal failure state as absent.
+- Terminal failure records for deployments with no previous state now record zero rollback attempts because no rollback was attempted.
+- Rollback commits no longer clear terminal failure state, so a failed desired digest remains blocked even after rollback restores the previous known-good deployment.
+- Commit failures during deployment or rollback now surface as deployment errors instead of being reported as successful deployments.
+- Backoff delay calculation now clamps safely before duration conversion and caps jittered delays to the configured max interval.
+
+### Upgrade Notes
+
+- Existing central state files remain compatible with the new state layout. Terminal failure state is stored as an optional `terminal_failure` field and is omitted when absent.
+- `DeploymentSnapshot` and `TerminalFailureState` no longer carry `workload_name`; the outer `DeploymentState.workload_name` identifies the workload.
+
 ## v0.4.1 - 2026-10-08
 
 ### Changed

README.md

@@ -96,6 +96,13 @@ contains one workload.
 Minimal example:
 
 ```yaml
+deployment_retry:
+  max_retry_forward_deployment: 3
+  max_retry_rollback: 2
+  initial_interval: 1s
+  max_interval: 30s
+  multiplier: 2
+
 workloads:
   - name: my-app
     release_prefix: my-app-prod
@@ -168,6 +175,36 @@ health_check:
   timeout: 5s
 ```
 
+## Deployment Retries
+
+By default, `reconD` keeps the original single-attempt behavior:
+
+```yaml
+deployment_retry:
+  max_retry_forward_deployment: 1
+  max_retry_rollback: 1
+  initial_interval: 1s
+  max_interval: 30s
+  multiplier: 2
+```
+
+Increase `max_retry_forward_deployment` to retry transient strategy,
+`deploy_command`, or health-check failures before rollback. Rollback starts only
+after the forward deployment attempts are exhausted. Increase
+`max_retry_rollback` to retry rollback failures. Retry waits use exponential
+backoff with jitter and stop promptly when the agent context is canceled.
+
+Set `deployment_retry` at the top level to define defaults for every workload in
+the config. A workload can include its own `deployment_retry` section to
+override only the fields that differ from the top-level defaults.
+
+If forward deployment attempts are exhausted, `reconD` persists a terminal
+failure for that workload and manifest digest in the central state file. When a
+previous deployment exists, rollback still runs to recover the workload, but a
+successful rollback does not make the failed desired digest eligible for
+automatic retry. Future reconciliation cycles skip that same digest until the
+terminal failure is cleared from state. A newer desired digest is not blocked.
+
 ## State Files
 
 State for all workloads is stored in one deployments state file. Set it with
@@ -288,6 +325,11 @@ Common optional fields:
 - `health_check.interval`: defaults to `10s`.
 - `health_check.timeout`: defaults to `5s`.
 - `labels`: labels used to scope Docker image pruning.
+- `deployment_retry.max_retry_forward_deployment`: total forward deployment attempts before rollback; defaults to `1`.
+- `deployment_retry.max_retry_rollback`: total rollback recovery attempts after forward deployment is exhausted; defaults to `1`.
+- `deployment_retry.initial_interval`: first retry backoff delay; defaults to `1s`.
+- `deployment_retry.max_interval`: maximum retry backoff delay; defaults to `30s`.
+- `deployment_retry.multiplier`: exponential backoff multiplier; defaults to `2`.
 
 ## Design Rule
 

config/config.go

@@ -4,6 +4,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
@@ -16,6 +17,10 @@ type Config struct {
 	// Log is optional and controls reconD's process-level logging output.
 	Log Log `yaml:"log"`
 
+	// DeploymentRetry is optional and provides default retry behavior for every
+	// workload in this config. Workload-level deployment_retry fields override it.
+	DeploymentRetry DeploymentRetry `yaml:"deployment_retry"`
+
 	// Workloads is the required list of container workloads reconD should watch
 	// and reconcile. Configure one entry for each independently deployed
 	// service.
@@ -77,6 +82,10 @@ type Workload struct {
 	// image tag.
 	Strategy Strategy `yaml:"strategy"`
 
+	// DeploymentRetry bounds automatic deployment and rollback attempts for one
+	// desired deployment identity.
+	DeploymentRetry DeploymentRetry `yaml:"deployment_retry"`
+
 	// Labels is optional and scopes Docker image pruning to images with these
 	// labels after a successful deployment.
 	Labels map[string]string `yaml:"labels"`
@@ -112,6 +121,25 @@ type Strategy struct {
 	ImageTagKey string `yaml:"image_tag_key"`
 }
 
+type DeploymentRetry struct {
+	// MaxRetryForwardDeployment is the total number of forward deployment
+	// attempts before rollback is considered. Defaults to 1.
+	MaxRetryForwardDeployment int `yaml:"max_retry_forward_deployment"`
+
+	// MaxRetryRollback is the total number of rollback attempts before the
+	// desired deployment is marked terminally failed. Defaults to 1.
+	MaxRetryRollback int `yaml:"max_retry_rollback"`
+
+	// InitialInterval is the first delay before retrying. Defaults to 1s.
+	InitialInterval Duration `yaml:"initial_interval"`
+
+	// MaxInterval caps exponential backoff delays. Defaults to 30s.
+	MaxInterval Duration `yaml:"max_interval"`
+
+	// Multiplier controls exponential backoff growth. Defaults to 2.
+	Multiplier float64 `yaml:"multiplier"`
+}
+
 type StrategyType string
 
 const (
@@ -165,7 +193,9 @@ func Load(path string) (*Config, error) {
 		return nil, err
 	}
 
+	cfg.DeploymentRetry.fillDefaultOnZeroValues()
 	for i := range cfg.Workloads {
+		cfg.Workloads[i].DeploymentRetry = mergeDeploymentRetry(cfg.DeploymentRetry, cfg.Workloads[i].DeploymentRetry)
 		cfg.Workloads[i].fillDefaultOnZeroValues()
 	}
 
@@ -212,12 +242,68 @@ func (w *Workload) fillDefaultOnZeroValues() {
 	if w.Strategy.Type == StrategyEnvFile && w.Strategy.ImageTagKey == "" {
 		w.Strategy.ImageTagKey = "IMAGE_TAG"
 	}
+	if w.DeploymentRetry.MaxRetryForwardDeployment == 0 {
+		w.DeploymentRetry.MaxRetryForwardDeployment = 1
+	}
+	if w.DeploymentRetry.MaxRetryRollback == 0 {
+		w.DeploymentRetry.MaxRetryRollback = 1
+	}
+	if w.DeploymentRetry.InitialInterval.Duration == 0 {
+		w.DeploymentRetry.InitialInterval.Duration = time.Second
+	}
+	if w.DeploymentRetry.MaxInterval.Duration == 0 {
+		w.DeploymentRetry.MaxInterval.Duration = 30 * time.Second
+	}
+	if w.DeploymentRetry.Multiplier == 0 {
+		w.DeploymentRetry.Multiplier = 2
+	}
+}
+
+func (r *DeploymentRetry) fillDefaultOnZeroValues() {
+	if r.MaxRetryForwardDeployment == 0 {
+		r.MaxRetryForwardDeployment = 1
+	}
+	if r.MaxRetryRollback == 0 {
+		r.MaxRetryRollback = 1
+	}
+	if r.InitialInterval.Duration == 0 {
+		r.InitialInterval.Duration = time.Second
+	}
+	if r.MaxInterval.Duration == 0 {
+		r.MaxInterval.Duration = 30 * time.Second
+	}
+	if r.Multiplier == 0 {
+		r.Multiplier = 2
+	}
+}
+
+func mergeDeploymentRetry(defaults, override DeploymentRetry) DeploymentRetry {
+	merged := defaults
+	if override.MaxRetryForwardDeployment != 0 {
+		merged.MaxRetryForwardDeployment = override.MaxRetryForwardDeployment
+	}
+	if override.MaxRetryRollback != 0 {
+		merged.MaxRetryRollback = override.MaxRetryRollback
+	}
+	if override.InitialInterval.Duration != 0 {
+		merged.InitialInterval = override.InitialInterval
+	}
+	if override.MaxInterval.Duration != 0 {
+		merged.MaxInterval = override.MaxInterval
+	}
+	if override.Multiplier != 0 {
+		merged.Multiplier = override.Multiplier
+	}
+	return merged
 }
 
 func (cfg Config) validate() error {
 	if len(cfg.Workloads) == 0 {
 		return fmt.Errorf("at least one workload must be configured")
 	}
+	if err := cfg.DeploymentRetry.validate(); err != nil {
+		return err
+	}
 
 	workloadNames := make(map[string]struct{}, len(cfg.Workloads))
 	for i, workload := range cfg.Workloads {
@@ -251,7 +337,10 @@ func (w Workload) validate() error {
 	if err := w.HealthCheck.validate(); err != nil {
 		return err
 	}
-	return w.Strategy.validate()
+	if err := w.Strategy.validate(); err != nil {
+		return err
+	}
+	return w.DeploymentRetry.validate()
 }
 
 func (h HealthCheck) validate() error {
@@ -287,6 +376,28 @@ func (s Strategy) validate() error {
 	return nil
 }
 
+func (r DeploymentRetry) validate() error {
+	if r.MaxRetryForwardDeployment < 1 {
+		return errors.New("deployment_retry.max_retry_forward_deployment must be at least 1")
+	}
+	if r.MaxRetryRollback < 1 {
+		return errors.New("deployment_retry.max_retry_rollback must be at least 1")
+	}
+	if r.InitialInterval.Duration <= 0 {
+		return errors.New("deployment_retry.initial_interval must be greater than 0")
+	}
+	if r.MaxInterval.Duration <= 0 {
+		return errors.New("deployment_retry.max_interval must be greater than 0")
+	}
+	if r.MaxInterval.Duration < r.InitialInterval.Duration {
+		return errors.New("deployment_retry.max_interval must be greater than or equal to initial_interval")
+	}
+	if math.IsNaN(r.Multiplier) || math.IsInf(r.Multiplier, 0) || r.Multiplier < 1 {
+		return errors.New("deployment_retry.multiplier must be a finite number at least 1")
+	}
+	return nil
+}
+
 func (d *Duration) UnmarshalYAML(value *yaml.Node) error {
 	if value.ShortTag() == "!!str" {
 		var s string