fix(reconcile): isolate current failure during rollback health

docs/EXPLICIT-DRAINED-RECOVERY.md

@@ -66,6 +66,21 @@ health report and matching process exit are required for each probe (0 for
 healthy/maintenance, 1 for warning, 2 for unhealthy). Missing output, stale output,
 execution failure or inconsistent exit/status cannot verify rollback. A rollback
 whose health is not drained maintenance/warning is not reported verified.
+After a confirmed LKG restoration, the manager must persist and read back a
+nonterminal reconciliation record identifying the rejected desired commit and
+the restored applied commit before the fresh rollback probe. This isolates the
+current transaction's failure from that probe without suppressing independent
+host-health checks or advancing the last-success timestamp. A missing or
+unverified record cannot authorize rollback success. The final transaction still
+reports failure or `rolled_back`; ordinary health continues to treat those
+terminal outcomes as critical, even after a healthy restoration. Terminal
+`rolled_back` publication must itself pass write/readback verification before
+any rollback completion indication. If it fails, the manager attempts one
+checked `failed` record retaining the attempted and restored identities. If that
+fallback is also unverified, it stops explicitly for operator intervention and
+emits no completion indication. Storage failure can leave a nonterminal record;
+this is unverified state, not a claim that ordinary health is durably critical.
+Write/readback verification is not a power-loss durability guarantee.
 Inspect the protected reconcile state and installer diagnostics after every
 nonzero exit. Failed recovery must remain held; admission needs separate review
 and authorization. This command does not establish active-pilot evidence,
@@ -88,7 +103,10 @@ template publication, private adoption and live-host acceptance are separate gat
 ## Local regression boundary
 
 `python3 -I -B scripts/test_explicit_recovery.py` uses temporary repositories and
-command doubles for authentication, installer, health and policy. It needs Linux
+command doubles for authentication, installer, host collection and policy. The
+rollback-state regressions use the production reconciliation-state reader and
+health evaluator against persisted fixture records, including independent
+critical host-health and state-persistence failures. The suite needs Linux
 Bash, Git, Python and flock, but no fleet credentials, Docker or live host. It
 proves orchestration and fail-closed routing, not actual image builds, installer
 transactions, systemd behavior or host acceptance. Complete exact-source/base

scripts/remote-reconcile.sh

@@ -94,9 +94,9 @@ require_commands() {
 
 save_reconcile_state() {
   local status=${1:-} desired_commit=${2:-} applied_commit=${3:-} health=${4:-} message=${5:-} mark_success=${6:-false}
-  local transaction_phase=${7:-} failure_reason=${8:-}
+  local transaction_phase=${7:-} failure_reason=${8:-} strict=${9:-false} write_status=0
   install -d -m 0700 "$reconcile_state_dir"
-  python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || true
+  python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || write_status=$?
 import json, os, sys, tempfile
 
 path = sys.argv[1]
@@ -134,6 +134,31 @@ except:
     os.unlink(tmp, missing_ok=True)
     raise
 PY
+  [[ "$strict" == true ]] || return 0
+  ((write_status == 0)) || return 1
+  python3 -c '
+import json, sys
+path, status, desired, applied, health, message, phase, reason = sys.argv[1:]
+value = json.load(open(path, encoding="utf-8"))
+expected = {"status": status, "desired_commit": desired, "applied_commit": applied,
+            "health": health, "message": message, "transaction_phase": phase, "failure_reason": reason}
+if not isinstance(value, dict) or any(value.get(key) != item for key, item in expected.items()):
+    raise SystemExit(1)
+' "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$transaction_phase" "$failure_reason" 2>/dev/null
+}
+
+finalize_rollback_state() {
+  local health=$1 message=$2
+  if save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$health" "$message" false '' '' true; then
+    return 0
+  fi
+  if save_reconcile_state 'failed' "$desired_commit" "$restored_config_ref" "$health" 'terminal rollback state persistence rejected' false '' '' true; then
+    printf 'RECONCILE_ERROR: terminal rollback state persistence rejected; failure record verified\n' >&2
+  else
+    printf 'STOP: reconciliation state persistence unverified; operator intervention required\n' >&2
+  fi
+  # Never use die here: another unchecked write cannot establish the outcome.
+  return 1
 }
 
 load_installed_state() {
@@ -651,9 +676,12 @@ PY
     *)
       save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}; final health rejected"
       apply_lkg || die "final health was ${health_status}; rollback to last-known-good also failed"
+      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good; checking rollback health' false '' '' true \
+        || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown'
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
-      verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified'
-      save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}"
+      verify_explicit_rollback_health "$rollback_health_status" \
+        || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
+      finalize_rollback_state "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}" || exit 2
       note "ROLLBACK_OK controller=${installed_controller} restored=${restored_config_ref}"
       exit 3
       ;;
@@ -685,10 +713,13 @@ finally:
 PY
       lkg_readback_matches "$restored_config_ref" || die 'baseline metadata restoration readback rejected'
       apply_lkg || die 'LKG persistence failed; drained baseline rollback failed'
+      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good after LKG persistence failure; checking rollback health' false '' '' true \
+        || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown'
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
-      verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified'
+      verify_explicit_rollback_health "$rollback_health_status" \
+        || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
       [[ "$rollback_health_status" == maintenance || "$rollback_health_status" == warning ]] || die 'LKG persistence failed; rollback health unverified'
-      save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" 'LKG persistence failed; restored drained baseline'
+      finalize_rollback_state "$rollback_health_status" 'LKG persistence failed; restored drained baseline' || exit 2
     fi
     exit 3
   fi