fix(purchase): allow Redshift re-drive behind the retry, lineage and journal gates

frontend/src/__tests__/history-retry-button.test.ts

@@ -550,6 +550,32 @@ describe('History inline Retry button (issue #47)', () => {
     expect(btn?.disabled).toBe(false);
   });
 
+  test('Redshift settle 409 with retry_after_seconds shows a wait toast, not a generic error', async () => {
+    (getCurrentUser as jest.Mock).mockReturnValue(ADMIN_USER);
+    (confirmDialog as jest.Mock).mockResolvedValue(true);
+    const settleError = Object.assign(new Error('this Redshift purchase failed too recently to retry safely; retry after the wait'), {
+      details: { retry_after_seconds: 90 },
+    });
+    (api.retryPurchase as jest.Mock).mockRejectedValue(settleError);
+    (api.getHistory as jest.Mock).mockResolvedValue({
+      summary: {},
+      purchases: [makeRow({ purchase_id: 'r-1', created_by_user_id: ADMIN_USER.id })],
+    });
+    console.error = jest.fn();
+
+    await loadHistory();
+    document.querySelector<HTMLButtonElement>('.history-retry-btn')?.click();
+    await flushPromises();
+
+    expect(showToast).toHaveBeenCalledWith(
+      expect.objectContaining({
+        kind: 'warning',
+        message: expect.stringContaining('Try again in 2 minutes'),
+      }),
+    );
+    expect(showToast).not.toHaveBeenCalledWith(expect.objectContaining({ kind: 'error' }));
+  });
+
   test('admin WITHOUT Purchaser membership does not see Retry on rows they did not create (CR #924 F5)', async () => {
     // Issue #923 + CR #924 F5: retry-any:purchases is carved out of
     // admin:*. canRetryFailedRow must gate on

frontend/src/history.ts

@@ -1647,6 +1647,16 @@ function wireRowActionHandlers(container: HTMLElement): void {
         const opsHint = typeof err.details?.['ops_hint'] === 'string' ? err.details['ops_hint'] : '';
         const retryAttemptN = typeof err.details?.['retry_attempt_n'] === 'number' ? err.details['retry_attempt_n'] : undefined;
         const threshold = typeof err.details?.['threshold'] === 'number' ? err.details['threshold'] : undefined;
+        // A Redshift purchase that failed minutes ago may still be settling at AWS (not overridable by force).
+        const retryAfterSeconds = typeof err.details?.['retry_after_seconds'] === 'number' ? err.details['retry_after_seconds'] : undefined;
+        if (!opsHint && retryAfterSeconds != null) {
+          const minutes = Math.max(1, Math.ceil(retryAfterSeconds / 60));
+          showToast({
+            message: `Retry is not possible yet: AWS may still be settling this purchase. Try again in ${minutes} minute${minutes === 1 ? '' : 's'}.`,
+            kind: 'warning',
+          });
+          return;
+        }
         let detailMessage = '';
         if (opsHint) {
           detailMessage = opsHint;

internal/api/handler_history.go

@@ -406,7 +406,7 @@ func executionToHistoryRow(exec config.PurchaseExecution, approver, createdByEma
 	// failed rows can carry a hint — the resolver returns "" for empty
 	// inputs, so this is safe to call unconditionally.
 	if exec.Status == "failed" {
-		row.OpsHint = resolveOpsHint(exec.Error)
+		row.OpsHint = purchase.ResolveOpsHint(exec.Error)
 		// A re-drive the backend will always refuse is permanent, so it wins
 		// over the operator-fixable failure-reason hint (which promises that
 		// fixing the config makes Retry work). Same call and input as the
@@ -421,12 +421,31 @@ func executionToHistoryRow(exec config.PurchaseExecution, approver, createdByEma
 			} else {
 				row.OpsHint = reason
 			}
+		} else {
+			applyRedshiftRetryHint(&row, &exec)
 		}
 	}
 	annotateHistoryRowByStatus(&row, exec, approver)
 	return row
 }
 
+// applyRedshiftRetryHint hides Retry for a Redshift row the gate refuses for good; the stamp age
+// is an app-clock hint, so near 24h Retry may show and then 409 from the database-clock gate.
+func applyRedshiftRetryHint(row *config.PurchaseHistoryRecord, exec *config.PurchaseExecution) {
+	if !purchase.IsRedshiftExecution(exec) {
+		return
+	}
+	var sinceStamp *time.Duration
+	if exec.FirstDispatchAt != nil {
+		age := max(0, time.Since(*exec.FirstDispatchAt))
+		sinceStamp = &age
+	}
+	if reason := purchase.RedshiftRetryPermanentRefusal(exec, sinceStamp); reason != "" {
+		row.RedriveUnsafe = true
+		row.OpsHint = reason
+	}
+}
+
 // definiteRefusalOpsHint replaces the duplicate-purchase warning when AWS
 // refused the purchase before making it.
 const definiteRefusalOpsHint = "AWS refused this purchase before it was made; start a new purchase after fixing the cause"

internal/api/handler_history_test.go

@@ -2192,13 +2192,12 @@ func TestExecutionToHistoryRow_RedriveUnsafe(t *testing.T) {
 		{"azure savings plans", failed("azure", "savingsplans", "boom"), true, ""},
 		{"unrecognized provider", failed("oracle", "compute", "boom"), true, ""},
 		{"failed aws ec2", failed("aws", "ec2", "boom"), true, ""},
-		{"failed aws redshift", failed("aws", "redshift", "boom"), true, ""},
 		{"failed mixed ec2+rds basket", config.PurchaseExecution{ExecutionID: "e-1", Status: "failed", Error: "boom", Recommendations: []config.RecommendationRecord{{Provider: "aws", Service: "rds"}, {Provider: "aws", Service: "ec2"}}}, true, ""},
 		{"pending ec2 gets no hint", withStatus("pending", config.RecommendationRecord{Provider: "aws", Service: "ec2"}), false, ""},
 		{"canceled ec2 gets no hint", withStatus("canceled", config.RecommendationRecord{Provider: "aws", Service: "ec2"}), false, ""},
 		{"aws rds is retryable", failed("aws", "rds", "boom"), false, ""},
 		{"refusal wins over failure-reason hint", failed("azure", "savingsplans", persistent), true, ""},
-		{"failure-reason hint kept when re-drive is safe", failed("aws", "rds", persistent), false, resolveOpsHint(persistent)},
+		{"failure-reason hint kept when re-drive is safe", failed("aws", "rds", persistent), false, purchase.ResolveOpsHint(persistent)},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -2214,6 +2213,46 @@ func TestExecutionToHistoryRow_RedriveUnsafe(t *testing.T) {
 	}
 }
 
+// The Redshift hint is a presentation hint on the app clock; the Retry gate stays authoritative.
+func TestExecutionToHistoryRow_RedshiftHistoryHintUsesAppClock(t *testing.T) {
+	const legacyArmedRefusal = "refusing to execute retry attempt 1: AWS EC2 and Redshift purchases have no provider-side duplicate guard, and a lost response means the earlier attempt may already have bought the commitment. Check EC2 Reserved Instances or Redshift Reserved Nodes in the target account and region; if it is missing, start a new purchase instead of retrying"
+	stampedAgo := func(d time.Duration) *time.Time { ts := time.Now().Add(-d); return &ts }
+	row := func(status string, attempt int, noSubmission bool, stamp *time.Time, errMsg string) config.PurchaseExecution {
+		return config.PurchaseExecution{
+			ExecutionID: "rs-1", Status: status, Error: errMsg, RetryAttemptN: attempt, NoSubmission: noSubmission, FirstDispatchAt: stamp,
+			Recommendations: []config.RecommendationRecord{{Provider: "aws", Service: "redshift", Region: "r", Selected: true}},
+		}
+	}
+	tests := []struct {
+		name       string
+		exec       config.PurchaseExecution
+		wantReason string
+	}{
+		{"stampless legacy failed row", row("failed", 1, false, nil, "boom"), "no first dispatch time recorded"},
+		{"stampless attempt 0 that sent", row("failed", 0, false, nil, "boom"), "no first dispatch time recorded"},
+		{"legacy armed refusal row", row("failed", 1, false, nil, legacyArmedRefusal), "no first dispatch time recorded"},
+		{"past the 24h horizon", row("failed", 1, false, stampedAgo(25*time.Hour), "boom"), "at least 24h ago"},
+		{"inside the horizon stays retryable", row("failed", 1, false, stampedAgo(time.Hour), "boom"), ""},
+		{"never submitted attempt 0 skips the stamp rules", row("failed", 0, true, nil, "boom"), ""},
+		{"never submitted attempt 0 past the horizon", row("failed", 0, true, stampedAgo(48*time.Hour), "boom"), ""},
+		{"failure-reason hint kept inside the horizon", row("failed", 1, false, stampedAgo(time.Hour), "FROM_EMAIL not configured"), ""},
+		{"pending row gets no hint", row("pending", 1, false, nil, ""), ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := executionToHistoryRow(tt.exec, "", "")
+			if tt.wantReason == "" {
+				assert.False(t, got.RedriveUnsafe)
+				assert.Equal(t, purchase.ResolveOpsHint(tt.exec.Error), got.OpsHint)
+				return
+			}
+			assert.True(t, got.RedriveUnsafe)
+			assert.Contains(t, got.OpsHint, tt.wantReason)
+		})
+	}
+	assert.Equal(t, legacyArmedRefusal, purchase.SanitizeStoredError(legacyArmedRefusal), "the legacy stored error still renders next to the new hint")
+}
+
 // TestHandler_getHistory_ProjectsCreatedBySystem: the History UI needs the #233 marker.
 func TestHandler_getHistory_ProjectsCreatedBySystem(t *testing.T) {
 	ctx := context.Background()

internal/api/handler_purchases.go

@@ -2358,46 +2358,6 @@ func (h *Handler) requireSession(ctx context.Context, req *events.LambdaFunction
 // they accumulate dozens of dead retry rows.
 const retryThreshold = 5
 
-// persistentFailureHints maps known-persistent failure substrings to
-// the operator-actionable hint surfaced on the History row in place of
-// the Retry button (issue #47, Q3). The match is a case-INsensitive
-// substring contains check (see resolveOpsHint) — wording variations
-// like "ses sandbox" / "SES Sandbox" all hit. Keep needles short and
-// distinctive so legitimate transient failures don't accidentally
-// substring-match a persistent hint and lock the user out.
-//
-// Membership criteria: only failures that NO retry can possibly fix —
-// they require an operator to change configuration or unblock an
-// upstream resource. SES sandbox / unverified domain / missing
-// FROM_EMAIL all fit; a transient SES throttle does NOT (the next
-// retry might succeed). When in doubt, leave it out — false-negatives
-// cost a wasted retry; false-positives lock the user out of an
-// actionable button entirely.
-var persistentFailureHints = map[string]string{
-	"FROM_EMAIL not configured": "Set FROM_EMAIL tfvar then retry",
-	"SES sandbox":               "Move SES out of sandbox or verify recipient, then retry",
-	"SES domain not verified":   "Verify SES domain in AWS console, then retry",
-	"IAM denied":                "Grant the deploy role missing IAM permission, then retry",
-}
-
-// resolveOpsHint returns a non-empty hint when the failure reason
-// matches a known-persistent pattern, else empty. Match is case-
-// insensitive substring contains so backend wording variations don't
-// silently slip through ("ses sandbox" / "SES sandbox" / "SES Sandbox"
-// all hit).
-func resolveOpsHint(failureReason string) string {
-	if failureReason == "" {
-		return ""
-	}
-	lower := strings.ToLower(failureReason)
-	for needle, hint := range persistentFailureHints {
-		if strings.Contains(lower, strings.ToLower(needle)) {
-			return hint
-		}
-	}
-	return ""
-}
-
 // retryPurchase creates a new execution from a *failed* execution's
 // stored Recommendations slice (issue #47). The original failed row
 // keeps its `failed` status as a historical record and gains a
@@ -2601,12 +2561,12 @@ func checkRetryEligibilityGates(failedExec *config.PurchaseExecution, req *event
 	// potentially a re-drive. purchase.RedriveRefusalReason is the same
 	// predicate the reaper's automatic re-drive gates on: it returns a
 	// reason exactly when the provider offers nothing that would collapse
-	// the second attempt onto the first, which today means AWS EC2 and
-	// Redshift (no client token; a lost response is reported as outcome
-	// unknown), Azure savings-plans (no server-side idempotency key,
-	// timestamp-derived order alias) plus any provider the predicate does
-	// not recognize. A failed EC2 or Redshift row therefore cannot be
-	// retried; the operator checks the account and starts a new purchase.
+	// the second attempt onto the first, which today means AWS EC2 (no
+	// client token; a lost response is reported as outcome unknown), Azure
+	// savings-plans (no server-side idempotency key, timestamp-derived order
+	// alias) plus any provider the predicate does not recognize. A failed EC2
+	// row therefore cannot be retried; the operator checks the account and
+	// starts a new purchase. Redshift is gated by checkRedshiftRetryGate.
 	// If plan steps ever carry recommendations (platform#609), a failed EC2
 	// step will be un-retryable too and needs its own recovery path.
 	//
@@ -2648,7 +2608,7 @@ func checkRetryEligibilityGates(failedExec *config.PurchaseExecution, req *event
 	// Persistent-failure block (Q3). Surfaces the ops_hint via the
 	// API for stale-cache callers; the History UI already shows it
 	// inline in place of the Retry button.
-	if hint := resolveOpsHint(failedExec.Error); hint != "" {
+	if hint := purchase.ResolveOpsHint(failedExec.Error); hint != "" {
 		return NewClientErrorWithDetails(409,
 			"this failure is operator-fixable; retrying without changing configuration will fail again",
 			map[string]any{"ops_hint": hint, "failure_reason": purchase.SanitizeStoredError(failedExec.Error)})

internal/api/handler_purchases_redshift_retry_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/LeanerCloud/cloud-commitments-platform/internal/config"
+	"github.com/aws/aws-lambda-go/events"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -100,17 +101,17 @@ func TestCheckRedshiftRetryGate_ClockErrorFailsClosed(t *testing.T) {
 	assert.False(t, isClient, "a store failure is a 500, not a refusal the UI would render as permanent")
 }
 
-// Until the old guard is lifted it refuses every Redshift Retry first, so the
-// new gate must not even read the clock.
-func TestLoadAndValidateRetry_OldRedriveRefusalStillWinsForRedshift(t *testing.T) {
+// The provider-level refusal no longer covers Redshift, so a Redshift Retry reaches the new gate.
+func TestLoadAndValidateRetry_RedshiftReachesTheNewGate(t *testing.T) {
 	failed := redshiftFailedRow(1, false)
 	session := &Session{UserID: retryCallerID, Email: "operator@example.com"}
 	h, store, _ := buildSessionRetryHandler(t, failed, session, false, true)
+	store.On("GetRedshiftRetryClock", mock.Anything, failed.ExecutionID).Return(config.RedshiftRetryClock{SinceUpdate: time.Hour}, nil)
 
 	_, _, err := h.loadAndValidateRetryRequest(context.Background(), sessionRetryReqWithForce(), failed.ExecutionID)
 
-	assertRedriveRefused(t, err, "Check EC2 Reserved Instances or Redshift Reserved Nodes")
-	store.AssertNotCalled(t, "GetRedshiftRetryClock", mock.Anything, mock.Anything)
+	assertRedriveRefused(t, err, "no first dispatch time recorded")
+	store.AssertCalled(t, "GetRedshiftRetryClock", mock.Anything, failed.ExecutionID)
 }
 
 func persistedSuccessor(t *testing.T, failed *config.PurchaseExecution) *config.PurchaseExecution {
@@ -156,3 +157,61 @@ func TestPersistRetryExecution_NoSubmissionAttempt0LeavesStampNil(t *testing.T)
 func TestPersistRetryExecution_StamplessPredecessorStaysStampless(t *testing.T) {
 	assert.Nil(t, persistedSuccessor(t, redshiftFailedRow(1, false)).FirstDispatchAt)
 }
+
+// retryRedshiftViaHandler drives the whole retryPurchase handler for a failed Redshift row.
+func retryRedshiftViaHandler(t *testing.T, failed *config.PurchaseExecution, clock config.RedshiftRetryClock, req *events.LambdaFunctionURLRequest) (saved []*config.PurchaseExecution, err error) {
+	t.Helper()
+	session := &Session{UserID: retryCallerID, Email: "operator@example.com"}
+	h, store, _ := buildSessionRetryHandler(t, failed, session, false, true)
+	store.On("GetRedshiftRetryClock", mock.Anything, failed.ExecutionID).Return(clock, nil)
+	store.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).
+		Run(func(args mock.Arguments) {
+			snap := *args.Get(1).(*config.PurchaseExecution)
+			saved = append(saved, &snap)
+		}).
+		Return(nil).Maybe()
+	store.On("LinkRetryExecutionAtomic", mock.Anything, mock.Anything, failed.ExecutionID, mock.AnythingOfType("string")).Return(true, nil).Maybe()
+
+	_, err = h.retryPurchase(context.Background(), req, failed.ExecutionID)
+	return saved, err
+}
+
+func TestRetryRedshift_InsideSettleRefusedWithRetryAfter(t *testing.T) {
+	stamp := time.Now().Add(-time.Hour)
+	failed := redshiftFailedRow(1, false)
+	failed.FirstDispatchAt = &stamp
+
+	for _, req := range []*events.LambdaFunctionURLRequest{sessionRetryReq(), sessionRetryReqWithForce()} {
+		saved, err := retryRedshiftViaHandler(t, failed, config.RedshiftRetryClock{SinceUpdate: 4 * time.Minute, SinceStamp: ageOf(time.Hour)}, req)
+
+		ce, ok := IsClientError(err)
+		require.True(t, ok, "got %v", err)
+		assert.Equal(t, 409, ce.code)
+		assert.Equal(t, 360, ce.Details()["retry_after_seconds"])
+		assert.Equal(t, "360", ce.headers["Retry-After"])
+		assert.Empty(t, saved, "a refused retry persists no successor")
+	}
+}
+
+func TestRetryRedshift_AfterSettleCopiesStamp(t *testing.T) {
+	stamp := time.Now().Add(-time.Hour).UTC().Truncate(time.Microsecond)
+	failed := redshiftFailedRow(1, false)
+	failed.FirstDispatchAt = &stamp
+
+	saved, err := retryRedshiftViaHandler(t, failed, config.RedshiftRetryClock{SinceUpdate: 11 * time.Minute, SinceStamp: ageOf(time.Hour)}, sessionRetryReq())
+
+	require.NoError(t, err)
+	require.NotEmpty(t, saved)
+	require.NotNil(t, saved[0].FirstDispatchAt)
+	assert.True(t, stamp.Equal(*saved[0].FirstDispatchAt), "the successor keeps the lineage stamp")
+	assert.Equal(t, 2, saved[0].RetryAttemptN)
+}
+
+func TestRetryRedshift_LegacyStamplessRefused(t *testing.T) {
+	failed := redshiftFailedRow(1, false)
+
+	saved, err := retryRedshiftViaHandler(t, failed, config.RedshiftRetryClock{SinceUpdate: time.Hour}, sessionRetryReqWithForce())
+
+	assertRedriveRefused(t, err, "no first dispatch time recorded")
+	assert.Empty(t, saved)
+}

internal/api/handler_purchases_retry_redrive_test.go

@@ -299,16 +299,16 @@ func TestRetryOfFailedAzureReservationStillPurchasesOnce(t *testing.T) {
 		"the retry must fire exactly one purchase, under the predecessor's token so Azure's two-step lookup dedupes it")
 }
 
-// TestRetryOfFailedAWSEC2OrRedshiftRowIsRefused is the MON-02 platform
-// regression guard. EC2 and Redshift purchases take no client token, so the
-// provider sends them once and reports a lost response as "purchase outcome
-// unknown": the commitment may exist. Retrying such a row must not create a
-// successor that buys again, whatever spelling the row carries (the canonical
-// compute and data-warehouse slugs dispatch to the same clients as ec2 and
-// redshift) and with ?force=true too.
-func TestRetryOfFailedAWSEC2OrRedshiftRowIsRefused(t *testing.T) {
+// TestRetryOfFailedAWSEC2RowIsRefused is the MON-02 platform regression guard.
+// EC2 purchases take no client token, so the provider sends them once and
+// reports a lost response as "purchase outcome unknown": the commitment may
+// exist. Retrying such a row must not create a successor that buys again,
+// whatever spelling the row carries (the canonical compute slug dispatches to
+// the same client as ec2) and with ?force=true too. Redshift has its own gate
+// (handler_purchases_redshift_retry_test.go).
+func TestRetryOfFailedAWSEC2RowIsRefused(t *testing.T) {
 	for _, tc := range []struct{ provider, service string }{
-		{"aws", "ec2"}, {"aws", "compute"}, {"aws", "redshift"}, {"aws", "data-warehouse"}, {"", "ec2"},
+		{"aws", "ec2"}, {"aws", "compute"}, {"", "ec2"},
 	} {
 		tc := tc
 		t.Run(tc.provider+"/"+tc.service, func(t *testing.T) {
@@ -319,8 +319,8 @@ func TestRetryOfFailedAWSEC2OrRedshiftRowIsRefused(t *testing.T) {
 				tokens, err := purchasesFiredByRetry(t, failed, req)
 
 				assert.Empty(t, tokens,
-					"a retry of an EC2/Redshift row after an unknown outcome must buy nothing")
-				assertRedriveRefused(t, err, "Check EC2 Reserved Instances or Redshift Reserved Nodes")
+					"a retry of an EC2 row after an unknown outcome must buy nothing")
+				assertRedriveRefused(t, err, "Check EC2 Reserved Instances in the target account")
 			}
 		})
 	}