From 5a54b8e29f41a796992271664951ee75658525ad Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 17:44:36 -0500
Subject: [PATCH 01/10] test(reconcile): reproduce rollback health state
 poisoning

---
 scripts/test_explicit_recovery.py | 60 +++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index c1532a5f..a9d68c24 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -207,6 +207,66 @@ def test_pretransaction_denial_keeps_old_lkg(self):
         self.assertEqual(self._reconcile_state()["transaction_phase"], "pre_transaction")
         self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg)
 
+    def install_evaluated_health(self):
+        """Keep host collection doubled, but evaluate the live reconcile record."""
+        scripts = self.root / "scripts"
+        for source, destination in (("health.py", "production_health.py"),
+                                    ("status_auth.py", "status_auth.py")):
+            shutil.copy2(Path(__file__).with_name(source), scripts / destination)
+        (scripts / "health.py").write_text('''import json, os, sys
+from pathlib import Path
+from production_health import Thresholds, evaluate, _reconcile_state
+
+calls = Path(os.environ["FAKE_INSTALLER_LOG"]).read_text().splitlines()
+rollback = len(calls) > 1
+record = _reconcile_state(Path(os.environ["CI_FLEET_RECONCILE_STATE_DIR"]) / "state.json")
+snapshot = {
+    "controller_id": "example-ci-01", "desired_state": "drained",
+    "disks": {"root": {"used_percent": 20, "inode_used_percent": 10},
+              "docker": {"used_percent": 30, "inode_used_percent": 15}},
+    "memory_available_percent": 75, "swap_used_percent": 0, "recent_oom": False,
+    "docker_available": rollback or os.environ.get("REJECT_CANDIDATE_HEALTH") != "1",
+    "controller": {"state": "missing", "restart_count": 0, "oom_killed": False},
+    "configured_capacity": {"min": 0, "max": 1}, "effective_capacity": {"min": 0, "max": 0},
+    "managed": {"running": 0, "inactive": 0, "unhealthy": 0, "restarting": 0},
+    "stale": {"images": 0, "networks": 0, "volumes": 0, "build_cache_bytes": 0},
+    "services": {"cleanup": "ok", "drift": "ok"},
+    "timers": {"health": "ok", "cleanup": "ok", "drift": "ok", "updates": "ok"},
+    "pending_reboot": False, "failed_packages": False, "clock_synchronized": True,
+    "backup": "not_configured", "reconciliation": record,
+}
+if rollback and os.environ.get("REJECT_ROLLBACK_HEALTH") == "1":
+    snapshot["docker_available"] = False
+report = evaluate(snapshot, Thresholds())
+with open(os.environ["EVALUATED_HEALTH_LOG"], "a", encoding="utf-8") as handle:
+    handle.write(json.dumps({"rollback": rollback, "record": record, "report": report}) + "\\n")
+Path(sys.argv[sys.argv.index("--output") + 1]).write_text(json.dumps(report), encoding="utf-8")
+raise SystemExit(report["exit_code"])
+''', encoding="utf-8")
+        self.env["EVALUATED_HEALTH_LOG"] = str(self.td / "evaluated-health.jsonl")
+        directory = Path(self.env["CI_FLEET_RECONCILE_STATE_DIR"])
+        directory.mkdir(mode=0o700, exist_ok=True)
+        (directory / "state.json").write_text(json.dumps({"last_success_at": 1}))
+
+    def test_final_health_rollback_uses_production_reconciliation_evaluator(self):
+        self.install_evaluated_health()
+        self.env["REJECT_CANDIDATE_HEALTH"] = "1"
+        result = self.run_ref("--ref", self.candidate)
+        self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
+        self.assertIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr)
+        state = self._reconcile_state()
+        self.assertEqual(state["status"], "rolled_back")
+        self.assertEqual(state["applied_commit"], self.baseline)
+        self.assertEqual(state["last_success_at"], 1)
+        observations = [json.loads(line) for line in
+                        Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()]
+        self.assertEqual(len(observations), 2)
+        self.assertTrue(observations[-1]["rollback"])
+        self.assertEqual(observations[-1]["record"]["status"], "reconciling")
+        self.assertEqual(observations[-1]["record"]["applied_commit"], self.baseline)
+        self.assertEqual(observations[-1]["report"]["status"], "warning")
+        self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg)
+
     def test_final_health_failure_rolls_back_to_drained_baseline(self):
         self.env["FAKE_HEALTH_STATUS"] = "unhealthy"
         result = self.run_ref("--ref", self.candidate)

From fedc4abe057fe195d3721fd296d017e7ad37b2c3 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 17:54:11 -0500
Subject: [PATCH 02/10] fix(reconcile): isolate restored rollback health from
 current failure

---
 scripts/remote-reconcile.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 64f1ca3a..44e174dc 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -651,6 +651,7 @@ PY
     *)
       save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}; final health rejected"
       apply_lkg || die "final health was ${health_status}; rollback to last-known-good also failed"
+      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good; checking rollback health'
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
       verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified'
       save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}"

From b1b471f638ce97ab5a7cecbfbc23301196799353 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 17:55:35 -0500
Subject: [PATCH 03/10] test(reconcile): cover rollback health persistence and
 LKG failures

---
 scripts/test_explicit_recovery.py | 51 +++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index a9d68c24..e07590b6 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -267,6 +267,57 @@ def test_final_health_rollback_uses_production_reconciliation_evaluator(self):
         self.assertEqual(observations[-1]["report"]["status"], "warning")
         self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg)
 
+    def test_lkg_failure_rollback_uses_production_reconciliation_evaluator(self):
+        for failure in ("write", "missing_write", "readback"):
+            with self.subTest(failure=failure):
+                self.setUp()
+                self.install_evaluated_health()
+                self.env["DENY_LKG_REF"] = self.candidate
+                real_python = shutil.which("python3")
+                assert real_python is not None
+                write_outcome = "exit 1" if failure == "write" else "exit 0"
+                if failure == "readback":
+                    condition = '${1:-} == -c && ${3:-} == "$CI_FLEET_LKG_DIR/metadata.json" && ${5:-} == "$DENY_LKG_REF"'
+                    write_outcome = "exit 1"
+                else:
+                    condition = '${1:-} == - && ${2:-} == "$CI_FLEET_LKG_DIR/metadata.json" && ${4:-} == "$DENY_LKG_REF"'
+                self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
+if [[ {condition} ]]; then {write_outcome}; fi
+exec {shlex.quote(real_python)} "$@"
+''')
+                result = self.run_ref("--ref", self.candidate)
+                self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
+                state = self._reconcile_state()
+                self.assertEqual(state["status"], "rolled_back", result.stdout + result.stderr)
+                self.assertEqual(state["applied_commit"], self.baseline)
+                self.assertEqual(state["last_success_at"], 1)
+                self.assertNotIn("RECONCILE_OK", result.stdout)
+                self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg)
+                observations = [json.loads(line) for line in
+                                Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()]
+                self.assertTrue(observations[-1]["rollback"])
+                self.assertEqual(observations[-1]["record"]["status"], "reconciling")
+                self.assertEqual(observations[-1]["record"]["applied_commit"], self.baseline)
+                self.assertEqual(observations[-1]["report"]["status"], "warning")
+
+    def test_missing_rollback_state_persistence_cannot_verify_recovery(self):
+        self.install_evaluated_health()
+        self.env["REJECT_CANDIDATE_HEALTH"] = "1"
+        real_python = shutil.which("python3")
+        assert real_python is not None
+        self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == reconciling ]] && \
+   [[ $(wc -l < "$FAKE_INSTALLER_LOG") -gt 1 ]]; then
+  rm -f -- "$CI_FLEET_RECONCILE_STATE_DIR/state.json"
+  exit 1
+fi
+exec {shlex.quote(real_python)} "$@"
+''')
+        result = self.run_ref("--ref", self.candidate)
+        self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr)
+        self.assertNotEqual(self._reconcile_state()["status"], "rolled_back")
+
     def test_final_health_failure_rolls_back_to_drained_baseline(self):
         self.env["FAKE_HEALTH_STATUS"] = "unhealthy"
         result = self.run_ref("--ref", self.candidate)

From 95714be378c6799c81c960a2ff55e16ffca26f5e Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:00:42 -0500
Subject: [PATCH 04/10] test(reconcile): reject stale rollback state and
 critical host health

---
 scripts/test_explicit_recovery.py | 32 +++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index e07590b6..bdce5833 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -318,6 +318,38 @@ def test_missing_rollback_state_persistence_cannot_verify_recovery(self):
         self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr)
         self.assertNotEqual(self._reconcile_state()["status"], "rolled_back")
 
+    def test_stale_rollback_state_readback_cannot_verify_recovery(self):
+        self.install_evaluated_health()
+        self.env["REJECT_CANDIDATE_HEALTH"] = "1"
+        real_python = shutil.which("python3")
+        assert real_python is not None
+        self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == reconciling ]] && \
+   [[ $(wc -l < "$FAKE_INSTALLER_LOG") -gt 1 ]]; then
+  rm -f -- "$CI_FLEET_RECONCILE_STATE_DIR/state.json"
+  exit 0
+fi
+exec {shlex.quote(real_python)} "$@"
+''')
+        result = self.run_ref("--ref", self.candidate)
+        self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr)
+        self.assertNotEqual(self._reconcile_state()["status"], "rolled_back")
+
+    def test_production_rollback_health_still_rejects_independent_failure(self):
+        self.install_evaluated_health()
+        self.env.update(REJECT_CANDIDATE_HEALTH="1", REJECT_ROLLBACK_HEALTH="1")
+        result = self.run_ref("--ref", self.candidate)
+        self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertNotIn("ROLLBACK_OK", result.stdout)
+        self.assertEqual(self._reconcile_state()["status"], "failed")
+        observations = [json.loads(line) for line in
+                        Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()]
+        self.assertTrue(observations[-1]["rollback"])
+        self.assertEqual(observations[-1]["report"]["status"], "unhealthy")
+        self.assertIn("docker", {check["id"] for check in observations[-1]["report"]["checks"]
+                                 if check["status"] == "critical"})
+
     def test_final_health_failure_rolls_back_to_drained_baseline(self):
         self.env["FAKE_HEALTH_STATUS"] = "unhealthy"
         result = self.run_ref("--ref", self.candidate)

From 5baffba8cff96eefd6fd4bd09a67e8a88ceff456 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:07:55 -0500
Subject: [PATCH 05/10] test(reconcile): retain restored identity on rejected
 rollback health

---
 scripts/test_explicit_recovery.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index bdce5833..e129be7f 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -342,7 +342,10 @@ def test_production_rollback_health_still_rejects_independent_failure(self):
         result = self.run_ref("--ref", self.candidate)
         self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
         self.assertNotIn("ROLLBACK_OK", result.stdout)
-        self.assertEqual(self._reconcile_state()["status"], "failed")
+        record = self._reconcile_state()
+        self.assertEqual(record["status"], "failed")
+        self.assertEqual(record["applied_commit"], self.baseline)
+        self.assertEqual(record["last_success_at"], 1)
         observations = [json.loads(line) for line in
                         Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()]
         self.assertTrue(observations[-1]["rollback"])

From a34b73303dc241389f733f1942e59a57d7e835a1 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:18:09 -0500
Subject: [PATCH 06/10] fix(reconcile): verify rollback probe state without
 masking host failures

---
 docs/EXPLICIT-DRAINED-RECOVERY.md | 13 ++++++++++++-
 scripts/remote-reconcile.sh       | 26 +++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md
index 076196ca..b024417b 100644
--- a/docs/EXPLICIT-DRAINED-RECOVERY.md
+++ b/docs/EXPLICIT-DRAINED-RECOVERY.md
@@ -66,6 +66,14 @@ health report and matching process exit are required for each probe (0 for
 healthy/maintenance, 1 for warning, 2 for unhealthy). Missing output, stale output,
 execution failure or inconsistent exit/status cannot verify rollback. A rollback
 whose health is not drained maintenance/warning is not reported verified.
+After a confirmed LKG restoration, the manager must persist and read back a
+nonterminal reconciliation record identifying the rejected desired commit and
+the restored applied commit before the fresh rollback probe. This isolates the
+current transaction's failure from that probe without suppressing independent
+host-health checks or advancing the last-success timestamp. A missing or
+unverified record cannot authorize rollback success. The final transaction still
+reports failure or `rolled_back`; ordinary health continues to treat those
+terminal outcomes as critical, even after a healthy restoration.
 Inspect the protected reconcile state and installer diagnostics after every
 nonzero exit. Failed recovery must remain held; admission needs separate review
 and authorization. This command does not establish active-pilot evidence,
@@ -88,7 +96,10 @@ template publication, private adoption and live-host acceptance are separate gat
 ## Local regression boundary
 
 `python3 -I -B scripts/test_explicit_recovery.py` uses temporary repositories and
-command doubles for authentication, installer, health and policy. It needs Linux
+command doubles for authentication, installer, host collection and policy. The
+rollback-state regressions use the production reconciliation-state reader and
+health evaluator against persisted fixture records, including independent
+critical host-health and state-persistence failures. The suite needs Linux
 Bash, Git, Python and flock, but no fleet credentials, Docker or live host. It
 proves orchestration and fail-closed routing, not actual image builds, installer
 transactions, systemd behavior or host acceptance. Complete exact-source/base
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 44e174dc..2181a024 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -94,9 +94,9 @@ require_commands() {
 
 save_reconcile_state() {
   local status=${1:-} desired_commit=${2:-} applied_commit=${3:-} health=${4:-} message=${5:-} mark_success=${6:-false}
-  local transaction_phase=${7:-} failure_reason=${8:-}
+  local transaction_phase=${7:-} failure_reason=${8:-} strict=${9:-false} write_status=0
   install -d -m 0700 "$reconcile_state_dir"
-  python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || true
+  python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || write_status=$?
 import json, os, sys, tempfile
 
 path = sys.argv[1]
@@ -134,6 +134,17 @@ except:
     os.unlink(tmp, missing_ok=True)
     raise
 PY
+  [[ "$strict" == true ]] || return 0
+  ((write_status == 0)) || return 1
+  python3 -c '
+import json, sys
+path, status, desired, applied, health, message, phase, reason = sys.argv[1:]
+value = json.load(open(path, encoding="utf-8"))
+expected = {"status": status, "desired_commit": desired, "applied_commit": applied,
+            "health": health, "message": message, "transaction_phase": phase, "failure_reason": reason}
+if not isinstance(value, dict) or any(value.get(key) != item for key, item in expected.items()):
+    raise SystemExit(1)
+' "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$transaction_phase" "$failure_reason" 2>/dev/null
 }
 
 load_installed_state() {
@@ -651,9 +662,11 @@ PY
     *)
       save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}; final health rejected"
       apply_lkg || die "final health was ${health_status}; rollback to last-known-good also failed"
-      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good; checking rollback health'
+      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good; checking rollback health' false '' '' true \
+        || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown'
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
-      verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified'
+      verify_explicit_rollback_health "$rollback_health_status" \
+        || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
       save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}"
       note "ROLLBACK_OK controller=${installed_controller} restored=${restored_config_ref}"
       exit 3
@@ -686,8 +699,11 @@ finally:
 PY
       lkg_readback_matches "$restored_config_ref" || die 'baseline metadata restoration readback rejected'
       apply_lkg || die 'LKG persistence failed; drained baseline rollback failed'
+      save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good after LKG persistence failure; checking rollback health' false '' '' true \
+        || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown'
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
-      verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified'
+      verify_explicit_rollback_health "$rollback_health_status" \
+        || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
       [[ "$rollback_health_status" == maintenance || "$rollback_health_status" == warning ]] || die 'LKG persistence failed; rollback health unverified'
       save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" 'LKG persistence failed; restored drained baseline'
     fi

From df02d262bc51c7772f8c31e185152ad39f6db39b Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:25:52 -0500
Subject: [PATCH 07/10] test(reconcile): reject terminal rollback persistence
 failures

---
 scripts/test_explicit_recovery.py | 49 +++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index e129be7f..d8b62fdf 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -353,6 +353,55 @@ def test_production_rollback_health_still_rejects_independent_failure(self):
         self.assertIn("docker", {check["id"] for check in observations[-1]["report"]["checks"]
                                  if check["status"] == "critical"})
 
+    def assert_terminal_rollback_failure_is_recorded(self, *, lkg_failure):
+        for failure in ("write", "missing_write", "readback"):
+            with self.subTest(failure=failure, lkg_failure=lkg_failure):
+                self.setUp()
+                self.install_evaluated_health()
+                if not lkg_failure:
+                    self.env["REJECT_CANDIDATE_HEALTH"] = "1"
+                self.env.update(TERMINAL_FAILURE=failure,
+                                DENY_LKG_REF=self.candidate if lkg_failure else "")
+                real_python = shutil.which("python3")
+                assert real_python is not None
+                self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
+  exit 1
+fi
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == rolled_back ]]; then
+  case "$TERMINAL_FAILURE" in
+    write) exit 1 ;;
+    missing_write) exit 0 ;;
+  esac
+fi
+if [[ ${{1:-}} == -c && ${{3:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{4:-}} == rolled_back && "$TERMINAL_FAILURE" == readback ]]; then
+  exit 1
+fi
+exec {shlex.quote(real_python)} "$@"
+''')
+                result = self.run_ref("--ref", self.candidate)
+                self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+                self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr)
+                self.assertNotIn("RECONCILE_OK", result.stdout)
+                record = self._reconcile_state()
+                self.assertEqual(record["status"], "failed", result.stdout + result.stderr)
+                self.assertEqual(record["desired_commit"], self.candidate)
+                self.assertEqual(record["applied_commit"], self.baseline)
+                self.assertEqual(record["last_success_at"], 1)
+                self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg)
+                report = self.td / "ordinary-health.json"
+                ordinary = subprocess.run([real_python, str(self.root / "scripts/health.py"),
+                                           "--output", str(report)], env=self.env,
+                                          capture_output=True, text=True, timeout=10)
+                self.assertEqual(ordinary.returncode, 2, ordinary.stdout + ordinary.stderr)
+                self.assertEqual(json.loads(report.read_text())["status"], "unhealthy")
+
+    def test_final_health_terminal_persistence_failure_stays_critical(self):
+        self.assert_terminal_rollback_failure_is_recorded(lkg_failure=False)
+
+    def test_lkg_terminal_persistence_failure_stays_critical(self):
+        self.assert_terminal_rollback_failure_is_recorded(lkg_failure=True)
+
     def test_final_health_failure_rolls_back_to_drained_baseline(self):
         self.env["FAKE_HEALTH_STATUS"] = "unhealthy"
         result = self.run_ref("--ref", self.candidate)

From 858a20708bb2774df505cb432fe88c3b90733c87 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:33:41 -0500
Subject: [PATCH 08/10] test(reconcile): require explicit stop when terminal
 fallback is unverified

---
 scripts/test_explicit_recovery.py | 37 +++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index d8b62fdf..2d3d5204 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -402,6 +402,43 @@ def test_final_health_terminal_persistence_failure_stays_critical(self):
     def test_lkg_terminal_persistence_failure_stays_critical(self):
         self.assert_terminal_rollback_failure_is_recorded(lkg_failure=True)
 
+    def test_terminal_and_fallback_persistence_failure_requires_operator(self):
+        for lkg_failure in (False, True):
+            with self.subTest(lkg_failure=lkg_failure):
+                self.setUp()
+                self.install_evaluated_health()
+                if not lkg_failure:
+                    self.env["REJECT_CANDIDATE_HEALTH"] = "1"
+                self.env.update(DENY_LKG_REF=self.candidate if lkg_failure else "",
+                                TERMINAL_ATTEMPT=str(self.td / "terminal-attempt"))
+                real_python = shutil.which("python3")
+                assert real_python is not None
+                self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
+  exit 1
+fi
+if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" ]]; then
+  if [[ ${{3:-}} == rolled_back ]]; then
+    : > "$TERMINAL_ATTEMPT"
+    exit 1
+  fi
+  if [[ ${{3:-}} == failed && -f "$TERMINAL_ATTEMPT" ]]; then exit 1; fi
+fi
+exec {shlex.quote(real_python)} "$@"
+''')
+                result = self.run_ref("--ref", self.candidate)
+                self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+                self.assertIn("STOP: reconciliation state persistence unverified; operator intervention required",
+                              result.stderr)
+                self.assertNotIn("ROLLBACK_OK", result.stdout)
+                self.assertNotIn("RECONCILE_OK", result.stdout)
+                record = self._reconcile_state()
+                self.assertEqual(record["status"], "reconciling")
+                self.assertEqual(record["desired_commit"], self.candidate)
+                self.assertEqual(record["applied_commit"], self.baseline)
+                self.assertEqual(record["last_success_at"], 1)
+                self.assertTrue(Path(self.env["TERMINAL_ATTEMPT"]).is_file())
+
     def test_final_health_failure_rolls_back_to_drained_baseline(self):
         self.env["FAKE_HEALTH_STATUS"] = "unhealthy"
         result = self.run_ref("--ref", self.candidate)

From d01ada1bb6857d107714a9640c808419986de2f4 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:41:31 -0500
Subject: [PATCH 09/10] test(reconcile): scope terminal failure injection to
 selected LKG ref

---
 scripts/test_explicit_recovery.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py
index 2d3d5204..a8a87cdf 100644
--- a/scripts/test_explicit_recovery.py
+++ b/scripts/test_explicit_recovery.py
@@ -365,7 +365,7 @@ def assert_terminal_rollback_failure_is_recorded(self, *, lkg_failure):
                 real_python = shutil.which("python3")
                 assert real_python is not None
                 self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
-if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
+if [[ -n "$DENY_LKG_REF" && ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
   exit 1
 fi
 if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == rolled_back ]]; then
@@ -414,7 +414,7 @@ def test_terminal_and_fallback_persistence_failure_requires_operator(self):
                 real_python = shutil.which("python3")
                 assert real_python is not None
                 self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash
-if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
+if [[ -n "$DENY_LKG_REF" && ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then
   exit 1
 fi
 if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" ]]; then

From 278c805edd0fcbad3c061c1842ea06a20e84c239 Mon Sep 17 00:00:00 2001
From: Nick Foster <1572453+Nickfost@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:49:14 -0500
Subject: [PATCH 10/10] fix(reconcile): verify terminal rollback state and
 checked failure fallback

---
 docs/EXPLICIT-DRAINED-RECOVERY.md |  9 ++++++++-
 scripts/remote-reconcile.sh       | 18 ++++++++++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md
index b024417b..56c24bbf 100644
--- a/docs/EXPLICIT-DRAINED-RECOVERY.md
+++ b/docs/EXPLICIT-DRAINED-RECOVERY.md
@@ -73,7 +73,14 @@ current transaction's failure from that probe without suppressing independent
 host-health checks or advancing the last-success timestamp. A missing or
 unverified record cannot authorize rollback success. The final transaction still
 reports failure or `rolled_back`; ordinary health continues to treat those
-terminal outcomes as critical, even after a healthy restoration.
+terminal outcomes as critical, even after a healthy restoration. Terminal
+`rolled_back` publication must itself pass write/readback verification before
+any rollback completion indication. If it fails, the manager attempts one
+checked `failed` record retaining the attempted and restored identities. If that
+fallback is also unverified, it stops explicitly for operator intervention and
+emits no completion indication. Storage failure can leave a nonterminal record;
+this is unverified state, not a claim that ordinary health is durably critical.
+Write/readback verification is not a power-loss durability guarantee.
 Inspect the protected reconcile state and installer diagnostics after every
 nonzero exit. Failed recovery must remain held; admission needs separate review
 and authorization. This command does not establish active-pilot evidence,
diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh
index 2181a024..88b22498 100755
--- a/scripts/remote-reconcile.sh
+++ b/scripts/remote-reconcile.sh
@@ -147,6 +147,20 @@ if not isinstance(value, dict) or any(value.get(key) != item for key, item in ex
 ' "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$transaction_phase" "$failure_reason" 2>/dev/null
 }
 
+finalize_rollback_state() {
+  local health=$1 message=$2
+  if save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$health" "$message" false '' '' true; then
+    return 0
+  fi
+  if save_reconcile_state 'failed' "$desired_commit" "$restored_config_ref" "$health" 'terminal rollback state persistence rejected' false '' '' true; then
+    printf 'RECONCILE_ERROR: terminal rollback state persistence rejected; failure record verified\n' >&2
+  else
+    printf 'STOP: reconciliation state persistence unverified; operator intervention required\n' >&2
+  fi
+  # Never use die here: another unchecked write cannot establish the outcome.
+  return 1
+}
+
 load_installed_state() {
   if [[ ! -f "$state_file" ]]; then
     return 1
@@ -667,7 +681,7 @@ PY
       rollback_health_status=$(run_health_check "$temp_dir/health.json")
       verify_explicit_rollback_health "$rollback_health_status" \
         || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
-      save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}"
+      finalize_rollback_state "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}" || exit 2
       note "ROLLBACK_OK controller=${installed_controller} restored=${restored_config_ref}"
       exit 3
       ;;
@@ -705,7 +719,7 @@ PY
       verify_explicit_rollback_health "$rollback_health_status" \
         || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status"
       [[ "$rollback_health_status" == maintenance || "$rollback_health_status" == warning ]] || die 'LKG persistence failed; rollback health unverified'
-      save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" 'LKG persistence failed; restored drained baseline'
+      finalize_rollback_state "$rollback_health_status" 'LKG persistence failed; restored drained baseline' || exit 2
     fi
     exit 3
   fi