ci: preserve production deploy order while merge groups validate

.github/workflows/ci.yaml

@@ -4387,6 +4387,36 @@ jobs:
           kube_version="$(yq -er '.spec.cluster.kubernetesVersion | select(tag == "!!str" and . != "")' ksail.prod.yaml)"
           bash scripts/guard-pdb-selector-match.sh --kube-version "$kube_version" k8s
 
+  wait-for-production-turn:
+    name: 🧾 Wait for the production deployment turn
+    needs: [changes]
+    if: github.event_name == 'merge_group' && needs.changes.outputs.k8s == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      pull-requests: read
+    steps:
+      - name: 📑 Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          fetch-depth: 2
+
+      - name: ⚙️ Setup Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      # Wait outside the shared production lock so a later candidate cannot
+      # hold that lock while the first candidate still needs it to merge.
+      - name: 🧾 Wait until the candidate is first on current main
+        env:
+          GH_TOKEN: ${{ github.token }}
+          QUEUE_RECEIPT: ${{ needs.changes.outputs.queue_receipt }}
+        run: go run ./scripts/verify-merge-group-queued wait
+
   # Every merge-group validator that ci-required-checks enforces is listed here too.
   # A required validator the deploy does not wait for runs in parallel with it, so
   # production can receive a revision that validator then rejects (#4148).
@@ -4412,6 +4442,7 @@ jobs:
         validate-root-verify-merge-group,
         validate-rgd-templates-merge-group,
         validate-ghcr-fanout-merge-group,
+        wait-for-production-turn,
       ]
     if: github.event_name == 'merge_group' && needs.changes.outputs.k8s == 'true'
     runs-on: ubuntu-latest
@@ -4791,6 +4822,7 @@ jobs:
         validate-root-verify-merge-group,
         validate-rgd-templates-merge-group,
         validate-ghcr-fanout-merge-group,
+        wait-for-production-turn,
         deploy-prod,
       ]
     if: always()
@@ -4921,8 +4953,25 @@ jobs:
             ${{ needs.validate-root-verify-merge-group.result }}
             ${{ needs.validate-rgd-templates-merge-group.result }}
             ${{ needs.validate-ghcr-fanout-merge-group.result }}
+            ${{ needs.wait-for-production-turn.result }}
             ${{ needs.deploy-prod.result }}
 
+      - name: 🚀 Require the selected production deployment
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          K8S_CHANGED: ${{ needs.changes.outputs.k8s }}
+          DEPLOY_RESULT: ${{ needs.deploy-prod.result }}
+        run: |
+          case "$EVENT_NAME:$K8S_CHANGED" in
+            merge_group:true) expected=success ;;
+            merge_group:false|pull_request:true|pull_request:false) expected=skipped ;;
+            *) echo '::error::Cannot determine production deployment selection'; exit 1 ;;
+          esac
+          if [ "$DEPLOY_RESULT" != "$expected" ]; then
+            echo "::error::Production deployment result was $DEPLOY_RESULT; expected $expected"
+            exit 1
+          fi
+
       - name: 🛡️ Require successful repository guards
         env:
           REPOSITORY_GUARDS_RESULT: ${{ needs.validate-repository-guards.result }}

scripts/validate-dr-signing/main_test.go

@@ -535,6 +535,22 @@ func TestCDWiringRejectsEachAblation(t *testing.T) {
 	}
 }
 
+// mutateMergeQueueDeploy keeps negative controls on the production job even
+// when another job has the same condition or dependency.
+func mutateMergeQueueDeploy(workflow, old, replacement string) string {
+	const deployHeader = "\n  deploy-prod:\n"
+	const nextJob = "\n  merge-group-queue-membership:\n"
+	before, fromDeploy, found := strings.Cut(workflow, deployHeader)
+	if !found {
+		return workflow
+	}
+	deploy, after, found := strings.Cut(fromDeploy, nextJob)
+	if !found {
+		return workflow
+	}
+	return before + deployHeader + strings.Replace(deploy, old, replacement, 1) + nextJob + after
+}
+
 func TestMergeQueueProductionRoutesUseTheCheckedAction(t *testing.T) {
 	t.Parallel()
 
@@ -579,11 +595,10 @@ func TestMergeQueueProductionRoutesUseTheCheckedAction(t *testing.T) {
 		// signing validator runs inside `changes`, so the publishing job runs
 		// behind a gate that already failed while this check reports success.
 		"merge-queue deploy overrides a failed dependency with always()": func(s string) string {
-			return strings.Replace(
+			return mutateMergeQueueDeploy(
 				s,
 				"    if: github.event_name == 'merge_group' && needs.changes.outputs.k8s == 'true'",
 				"    if: always() && github.event_name == 'merge_group' && needs.changes.outputs.k8s == 'true'",
-				1,
 			)
 		},
 		// #2942 on the route production normally takes. Requiring the job to
@@ -1596,46 +1611,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) {
 		validatorLine = "          go run ./scripts/validate-dr-signing .github/workflows/dr-rebuild.yaml ksail.prod.yaml\n"
 		testLine      = "          go test ./scripts/validate-dr-signing\n"
 		gateJobHeader = "  validate-publication-contract:\n"
-		deployNeeds   = `    needs:
-      [
-        changes,
-        validate-repository-guards,
-        validate-independent-guards,
-        test-publish-release-recovery,
-        validate-publication-generator,
-        validate-go-regressions,
-        validate-webapp-compiler,
-        validate-consumer-discovery,
-        validate-floating-image-tags,
-        validate-helm-post-renderers,
-        validate-eks-authorization,
-        validate-publication-contract,
-        validate-matcher-efficacy,
-        validate-talos,
-        validate-root-verify-merge-group,
-        validate-rgd-templates-merge-group,
-        validate-ghcr-fanout-merge-group,
-      ]`
-		deployNeedsWithoutGate = `    needs:
-      [
-        changes,
-        validate-repository-guards,
-        validate-independent-guards,
-        test-publish-release-recovery,
-        validate-publication-generator,
-        validate-go-regressions,
-        validate-webapp-compiler,
-        validate-consumer-discovery,
-        validate-floating-image-tags,
-        validate-helm-post-renderers,
-        validate-eks-authorization,
-        validate-matcher-efficacy,
-        validate-talos,
-        validate-root-verify-merge-group,
-        validate-rgd-templates-merge-group,
-        validate-ghcr-fanout-merge-group,
-      ]`
-		healNeeds = `    needs:
+		healNeeds     = `    needs:
       [
         changes,
         deploy-prod,
@@ -1713,7 +1689,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) {
 		// are not redundant.
 		"deploy-prod no longer requires the gate job": {
 			func(s string) string {
-				return strings.Replace(s, deployNeeds, deployNeedsWithoutGate, 1)
+				return mutateMergeQueueDeploy(s, "        validate-publication-contract,\n", "")
 			},
 			"does not require validate-publication-contract",
 		},

scripts/validate-merge-group-heal/production_outcome_test.go

@@ -73,6 +73,8 @@ func TestProductionOutcomeMutations(t *testing.T) {
 	jobAnchor := "    # `true` only when the queue preflight refused"
 	stepsAnchor := "stopped_before_production }}\n    steps:\n      - name: 📑 Checkout\n        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1\n        with:\n"
 	setupAnchor := stepsAnchor + "          persist-credentials: false\n          fetch-depth: 0\n\n      - name: ⚙️ Setup Go\n        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0\n        with:\n"
+	receiptLine := "          QUEUE_RECEIPT: ${{ needs.changes.outputs.queue_receipt }}\n"
+	checkLine := "        run: go run ./scripts/verify-merge-group-queued check\n"
 	for _, tc := range []struct{ name, from, to string }{
 		{"output from another step", "${{ steps.production-outcome.outputs.stopped_before_production }}", "${{ steps.queue-preflight.outputs.stopped_before_production }}"},
 		{"extra output", "      stopped_before_production: ${{ steps.production-outcome.outputs.stopped_before_production }}", "      stopped_before_production: ${{ steps.production-outcome.outputs.stopped_before_production }}\n      other: 'true'"},
@@ -89,8 +91,8 @@ func TestProductionOutcomeMutations(t *testing.T) {
 		{"preflight is conditional", "        id: queue-preflight\n", "        id: queue-preflight\n        if: success()\n"},
 		{"preflight is suppressed", "        id: queue-preflight\n", "        id: queue-preflight\n        continue-on-error: true\n"},
 		{"preflight changes shell", "        run: go run ./scripts/verify-merge-group-queued check\n", "        shell: sh\n        run: go run ./scripts/verify-merge-group-queued check\n"},
-		{"preflight gains a variable", "          QUEUE_RECEIPT: ${{ needs.changes.outputs.queue_receipt }}\n        run: go run", "          QUEUE_RECEIPT: ${{ needs.changes.outputs.queue_receipt }}\n          GOFLAGS: -toolexec=./deploy\n        run: go run"},
-		{"preflight reads another receipt", "          QUEUE_RECEIPT: ${{ needs.changes.outputs.queue_receipt }}\n        run: go run", "          QUEUE_RECEIPT: forged\n        run: go run"},
+		{"preflight gains a variable", receiptLine + checkLine, receiptLine + "          GOFLAGS: -toolexec=./deploy\n" + checkLine},
+		{"preflight reads another receipt", receiptLine + checkLine, "          QUEUE_RECEIPT: forged\n" + checkLine},
 		{"preflight runs elsewhere", "        run: go run ./scripts/verify-merge-group-queued check\n", "        working-directory: other\n        run: go run ./scripts/verify-merge-group-queued check\n"},
 		{"workflow environment", "\njobs:\n", "\nenv:\n  BASH_ENV: ./deploy.sh\njobs:\n"},
 		{"workflow defaults", "\njobs:\n", "\ndefaults:\n  run:\n    working-directory: other\njobs:\n"},