fix(hotfix): self-heal a stale orphan env branch instead of fail-closing

e2e/harness/hotfix_actions.go

@@ -135,6 +135,19 @@ func (r *Runner) executeHotfixPlan(ctx context.Context, step *HotfixPlanStep) er
 	for name, job := range result.Jobs {
 		r.t.Logf("    - Job '%s': conclusion=%s", name, job.Conclusion)
 	}
+
+	// assert_branch_reset: true is a sharper signal than plan success alone; it
+	// proves the orphan self-heal diagnostic line was emitted, confirming
+	// ResetBranch fired rather than the plan succeeding for another reason
+	// (e.g. branch already at base, branch newly created).
+	if step.AssertBranchReset {
+		const marker = "cascade: orphan"
+		if !strings.Contains(result.Logs, marker) {
+			r.t.Logf("  HotfixPlan workflow logs:\n%s", result.Logs)
+			return fmt.Errorf("assert_branch_reset: expected orphan self-heal diagnostic in plan logs but it was not found")
+		}
+		r.t.Log("  HotfixPlan: orphan self-heal confirmed (branch_reset=true)")
+	}
 	return nil
 }
 

e2e/harness/multistep.go

@@ -106,10 +106,14 @@ type Step struct {
 // workflow's plan job. CommitRef is the trunk commit to plan a hotfix for and is
 // resolved via the execution context (falling back to a literal SHA).
 type HotfixPlanStep struct {
-	CommitRef     string `yaml:"commit_ref"`
-	TargetEnv     string `yaml:"target_env"`
-	DryRun        bool   `yaml:"dry_run,omitempty"`
-	ExpectFailure bool   `yaml:"expect_failure,omitempty"`
+	CommitRef        string `yaml:"commit_ref"`
+	TargetEnv        string `yaml:"target_env"`
+	DryRun           bool   `yaml:"dry_run,omitempty"`
+	ExpectFailure    bool   `yaml:"expect_failure,omitempty"`
+	// AssertBranchReset, when true, asserts that the plan workflow logged the
+	// orphan self-heal diagnostic line (branch_reset=true), confirming the heal
+	// fired rather than the plan merely succeeding for another reason.
+	AssertBranchReset bool `yaml:"assert_branch_reset,omitempty"`
 }
 
 // HotfixApplyStep defines a hotfix_apply action: a harness-driven cherry-pick of

e2e/scenarios/hotfix/hotfix-orphan-selfheal.yaml

@@ -0,0 +1,117 @@
+name: "Hotfix Orphan Self-Heal"
+description: |
+  Verifies the orphan self-heal path: an env/<env> integration branch left
+  behind by an interrupted hotfix (the branch exists but the environment never
+  recorded divergence) is force-reset back to its recorded base by a later
+  hotfix plan, instead of failing closed on the stale tip.
+
+  The branch is an orphan because the resolution PR was merged but the divergence
+  was never finalized, so state[test] reports no divergence while env/test leads
+  the recorded base. Trunk then advances past that tip. A new hotfix plan for
+  test, with the single-flight gate satisfied (no open resolution PR), proves the
+  branch is an abandoned orphan and resets it to the recorded base so the plan
+  succeeds rather than aborting.
+
+config:
+  trunk_branch: main
+  environments: [dev, test, prod]
+  builds:
+    - name: app
+      workflow: build.yaml
+      triggers: ["src/**"]
+  deploys:
+    - name: deploy-dev
+      workflow: deploy.yaml
+      triggers: ["src/**"]
+    - name: deploy-test
+      workflow: deploy.yaml
+      triggers: ["src/**"]
+    - name: deploy-prod
+      workflow: deploy.yaml
+      triggers: ["src/**"]
+
+steps:
+  - name: "Initial commit"
+    action: commit
+    commit:
+      message: "feat: add app"
+      files:
+        src/app.go: |
+          package main
+          func main() {}
+
+  - name: "Orchestrate trunk into dev/test"
+    action: orchestrate
+
+  - name: "Promote to establish a test/prod baseline"
+    action: promote
+    promote:
+      mode: default
+
+  - name: "Commit a trunk fix to hotfix into test"
+    action: commit
+    commit:
+      message: "fix: patch for test env"
+      files:
+        src/fix.go: |
+          package main
+          func patch() {}
+
+  - name: "Apply hotfix onto env/test, seeding the integration branch"
+    action: hotfix_apply
+    hotfix_apply:
+      target_env: test
+      commit_ref: commit2
+
+  # Merge the resolution PR so no open cascade-hotfix PR remains, but do NOT
+  # finalize: state[test] never records divergence. env/test now exists and
+  # leads the recorded base while the environment looks healthy. That is the
+  # orphan condition the self-heal targets.
+  - name: "Merge the resolution PR without finalizing divergence"
+    action: merge_pr
+    merge_pr:
+      label: cascade-hotfix
+
+  # Advance trunk and promote so state[test].SHA moves forward, past the orphan
+  # env/test tip. The recorded base the next hotfix derives now disagrees with
+  # the stale env/test branch.
+  - name: "Advance trunk normally"
+    action: commit
+    commit:
+      message: "chore: trunk advance"
+      files:
+        src/advance.go: |
+          package main
+          func advance() {}
+
+  - name: "Orchestrate the advance into dev/test"
+    action: orchestrate
+
+  - name: "Promote so test advances past the orphan tip"
+    action: promote
+    promote:
+      mode: default
+
+  - name: "Commit a new trunk fix to hotfix into test"
+    action: commit
+    commit:
+      message: "fix: second patch for test env"
+      files:
+        src/fix2.go: |
+          package main
+          func patch2() {}
+
+  # The plan runs for real (not a dry run). The generated plan job passes --repo,
+  # so the single-flight gate runs with a real checker; finding no open
+  # resolution PR it proves env/test is an abandoned orphan and force-resets it
+  # to the recorded base. Before the self-heal this plan would fail closed on the
+  # stale env/test tip; success proves the heal. assert_branch_reset sharpens
+  # the signal: it checks the diagnostic line emitted when ResetBranch fires,
+  # distinguishing a genuine self-heal from a plan that succeeds for another
+  # reason (branch already at base, branch freshly created, etc.).
+  - name: "Plan hotfix for test self-heals the orphan env branch"
+    action: hotfix_plan
+    hotfix_plan:
+      target_env: test
+      commit_ref: commit4
+      assert_branch_reset: true

internal/generate/hotfix.go

@@ -218,6 +218,17 @@ func (g *HotfixGenerator) writePlanJob(sb *strings.Builder) {
 	sb.WriteString("    name: Plan Hotfix\n")
 	sb.WriteString("    if: github.event_name == 'workflow_dispatch'\n")
 	sb.WriteString("    runs-on: ubuntu-latest\n")
+	// The plan job runs the single-flight gate via --repo and self-heals an
+	// abandoned env branch by force-pushing it back to its recorded base, so it
+	// needs contents: write. The single-flight lookup lists hotfix PRs, which
+	// requires pull-requests: read on a private repo. actions: read covers
+	// workflow introspection. These writes live on the plan job, not at the
+	// least-privilege top level.
+	writeJobPermissions(sb, "    ", [][2]string{
+		{"contents", "write"},
+		{"pull-requests", "read"},
+		{"actions", "read"},
+	})
 	sb.WriteString("    outputs:\n")
 	sb.WriteString("      branch: ${{ steps.plan.outputs.branch }}\n")
 	sb.WriteString("      fix_sha: ${{ steps.plan.outputs.fix_sha }}\n")
@@ -242,6 +253,9 @@ func (g *HotfixGenerator) writePlanJob(sb *strings.Builder) {
 	sb.WriteString("      - name: Plan hotfix\n")
 	sb.WriteString("        id: plan\n")
 	sb.WriteString("        env:\n")
+	// GH_TOKEN authenticates the single-flight REST API call the planner makes when
+	// --repo is set. Without it the lookup may fail on private repos and the plan aborts.
+	sb.WriteString("          GH_TOKEN: ${{ github.token }}\n")
 	sb.WriteString("          HOTFIX_COMMIT: ${{ github.event.inputs.commit }}\n")
 	sb.WriteString("          HOTFIX_TARGET_ENV: ${{ github.event.inputs.target_env }}\n")
 	sb.WriteString("          HOTFIX_DRY_RUN: ${{ github.event.inputs.dry_run }}\n")
@@ -250,6 +264,10 @@ func (g *HotfixGenerator) writePlanJob(sb *strings.Builder) {
 	fmt.Fprintf(sb, "            --config %s \\\n", g.getManifestFilePath())
 	sb.WriteString("            --commits \"$HOTFIX_COMMIT\" \\\n")
 	sb.WriteString("            --target-env \"$HOTFIX_TARGET_ENV\" \\\n")
+	// --repo wires the single-flight PR lookup to a real REST-backed checker.
+	// Without it the gate is inert (the no-op checker), which both skips the
+	// single-flight protection and, by design, disables orphan self-heal.
+	sb.WriteString("            --repo \"${{ github.repository }}\" \\\n")
 	sb.WriteString("            --dry-run=\"$HOTFIX_DRY_RUN\" \\\n")
 	sb.WriteString("            --gha-output\n")
 

internal/generate/hotfix_test.go

@@ -117,6 +117,14 @@ func TestHotfixGenerator_PlanJobChainOutputs(t *testing.T) {
 
 	assert.Contains(t, planJob, "env_sequence: ${{ steps.plan.outputs.env_sequence }}",
 		"plan job must expose the env_sequence chain order")
+
+	// The single-flight gate and orphan self-heal only run when --repo wires a
+	// real gh-backed checker; the plan job must pass the repository slug and carry
+	// contents: write so the self-heal force-push to origin can land.
+	assert.Contains(t, planJob, `--repo "${{ github.repository }}"`,
+		"plan job must pass --repo so the single-flight gate runs with a real checker")
+	assert.Contains(t, planJob, "contents: write",
+		"plan job must carry contents: write so the orphan self-heal can force-push env/<env>")
 	for _, key := range []string{
 		"commits_test: ${{ steps.plan.outputs.commits_test }}",
 		"no_op_test: ${{ steps.plan.outputs.no_op_test }}",

internal/hotfix/chain.go

@@ -35,6 +35,12 @@ type EnvPlan struct {
 	// NoOp is true when the whole requested set is already present in this env.
 	NoOp bool `json:"no_op"`
 
+	// BranchReset is true when the remote env/<env> branch existed at a stale tip
+	// but was force-reset back to BaseSHA by the orphan self-heal. It is set only
+	// when the env is not diverged and the single-flight gate ran with a real
+	// checker that found no open resolution PR.
+	BranchReset bool `json:"branch_reset"`
+
 	// ConflictExpected hints whether a cherry-pick is likely to conflict. The
 	// plan verb does not run the cherry-pick, so this is best-effort and false
 	// by default; the workflow is authoritative.
@@ -130,25 +136,39 @@ func commitPresentInEnv(fixSHA, baseSHA string, patches []string) (bool, error)
 	return already, nil
 }
 
-// verifyRemoteEnvTip fails the plan when the fetched remote env branch tip has
-// diverged from the recorded base SHA. When the remote ref is absent (the env
-// has never been hotfixed, so the apply job will create env/<env> at baseSHA)
-// there is nothing to diverge from and the check passes, leaving the normal
-// path untouched. This mirrors reconcileBranch's local-branch guard but runs
-// against the remote-tracking ref the generated hotfix workflow fetches, which
-// is the only env ref present in CI.
-func (p *Planner) verifyRemoteEnvTip(branch, baseSHA string) error {
+// verifyRemoteEnvTip reconciles the fetched remote env branch tip against the
+// recorded base SHA. When the remote ref is absent (the env has never been
+// hotfixed, so the apply job will create env/<env> at baseSHA) there is nothing
+// to diverge from and the check passes, leaving the normal path untouched. When
+// the tip has drifted it is either self-healed back to baseSHA or the plan
+// aborts fail-closed, following the same orphan-safety rule as reconcileBranch:
+// reset only when the env is not diverged AND the single-flight gate ran with a
+// real checker that found no open resolution PR. This mirrors reconcileBranch's
+// local-branch guard but runs against the remote-tracking ref the generated
+// hotfix workflow fetches, which is the only env ref present in CI.
+//
+// Returns whether the branch was (or, in dry-run, would be) reset.
+func (p *Planner) verifyRemoteEnvTip(branch, baseSHA string, diverged, singleFlightChecked bool) (reset bool, err error) {
 	tip, exists, err := p.gitRunner.RemoteBranchSHA(p.remote, branch)
 	if err != nil {
-		return fmt.Errorf("reading remote tip of %s: %w", branch, err)
+		return false, fmt.Errorf("reading remote tip of %s: %w", branch, err)
 	}
 	if !exists {
-		return nil
+		return false, nil
 	}
 	if tip != baseSHA {
-		return envTipDivergenceError(branch, tip, baseSHA)
+		if !diverged && singleFlightChecked {
+			if p.dryRun {
+				return true, nil
+			}
+			if err := p.gitRunner.ResetBranch(p.remote, branch, baseSHA); err != nil {
+				return false, fmt.Errorf("self-healing orphan %s to %s: %w", branch, short(baseSHA), err)
+			}
+			return true, nil
+		}
+		return false, envTipDivergenceError(branch, tip, baseSHA, diverged)
 	}
-	return nil
+	return false, nil
 }
 
 // PlanChain validates and computes the per-environment plan for elevating a set
@@ -190,20 +210,34 @@ func (p *Planner) PlanChain(refs []string, targetEnv string) (*PlanChainResult,
 		}
 		baseSHA := state.SHA
 
-		// Guard against a remote env branch that has drifted from recorded state
-		// before deriving a cherry-pick base from it. The base is state.SHA, but
-		// the apply job opens the resolution PR against the live env/<env> branch;
-		// if the fetched remote tip has diverged the cherry-pick lands on a stale
-		// base and the PR is unmergeable, surfacing only as a merge-poll timeout.
-		if err := p.verifyRemoteEnvTip(envBranch(env), baseSHA); err != nil {
+		// Single-flight gate per env, BEFORE the remote-tip reconciliation. The
+		// chain path historically skipped this gate; adding it here is what makes
+		// the per-env self-heal both enabled (a real checker can set
+		// singleFlightChecked) and safe (an open resolution PR aborts the plan
+		// exactly as the single-env path does, so a live hotfix is never reset).
+		diverged := state.IsDiverged()
+		singleFlightChecked, err := p.checkSingleFlight(envBranch(env))
+		if err != nil {
+			return nil, err
+		}
+
+		// Reconcile the remote env branch against recorded state before deriving a
+		// cherry-pick base from it. The base is state.SHA, but the apply job opens
+		// the resolution PR against the live env/<env> branch; if the fetched remote
+		// tip has diverged the cherry-pick lands on a stale base and the PR is
+		// unmergeable, surfacing only as a merge-poll timeout. An abandoned orphan
+		// tip is self-healed; an in-progress hotfix stays fail-closed.
+		reset, err := p.verifyRemoteEnvTip(envBranch(env), baseSHA, diverged, singleFlightChecked)
+		if err != nil {
 			return nil, err
 		}
 
 		ep := EnvPlan{
-			Env:     env,
-			Branch:  envBranch(env),
-			BaseSHA: baseSHA,
-			Commits: make([]string, 0, len(shas)),
+			Env:         env,
+			Branch:      envBranch(env),
+			BaseSHA:     baseSHA,
+			BranchReset: reset,
+			Commits:     make([]string, 0, len(shas)),
 		}
 		for _, sha := range shas {
 			present, err := commitPresentInEnv(sha, baseSHA, state.Patches)

internal/hotfix/chain_test.go

@@ -405,8 +405,14 @@ func TestPlanChain_RemoteEnvTipDiverged_FailsWithGuidance(t *testing.T) {
 		t.Errorf("error %q should report both the branch tip %s and the recorded SHA %s",
 			msg, short7(diverged), short7(base))
 	}
-	if !strings.Contains(strings.ToLower(msg), "replay") {
-		t.Errorf("error %q should give reconcile/replay guidance", msg)
+	// With no real PRChecker injected the single-flight gate did not run, so the
+	// planner must stay fail-closed (no self-heal) and surface actionable
+	// recovery guidance.
+	if !strings.Contains(msg, "abandoned hotfix branch") {
+		t.Errorf("error %q should describe the abandoned branch", msg)
+	}
+	if !strings.Contains(msg, "--repo") {
+		t.Errorf("error %q should point at re-running with --repo", msg)
 	}
 }