diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md index 076196ca..56c24bbf 100644 --- a/docs/EXPLICIT-DRAINED-RECOVERY.md +++ b/docs/EXPLICIT-DRAINED-RECOVERY.md @@ -66,6 +66,21 @@ health report and matching process exit are required for each probe (0 for healthy/maintenance, 1 for warning, 2 for unhealthy). Missing output, stale output, execution failure or inconsistent exit/status cannot verify rollback. A rollback whose health is not drained maintenance/warning is not reported verified. +After a confirmed LKG restoration, the manager must persist and read back a +nonterminal reconciliation record identifying the rejected desired commit and +the restored applied commit before the fresh rollback probe. This isolates the +current transaction's failure from that probe without suppressing independent +host-health checks or advancing the last-success timestamp. A missing or +unverified record cannot authorize rollback success. The final transaction still +reports failure or `rolled_back`; ordinary health continues to treat those +terminal outcomes as critical, even after a healthy restoration. Terminal +`rolled_back` publication must itself pass write/readback verification before +any rollback completion indication. If it fails, the manager attempts one +checked `failed` record retaining the attempted and restored identities. If that +fallback is also unverified, it stops explicitly for operator intervention and +emits no completion indication. Storage failure can leave a nonterminal record; +this is unverified state, not a claim that ordinary health is durably critical. +Write/readback verification is not a power-loss durability guarantee. Inspect the protected reconcile state and installer diagnostics after every nonzero exit. Failed recovery must remain held; admission needs separate review and authorization. This command does not establish active-pilot evidence, @@ -88,7 +103,10 @@ template publication, private adoption and live-host acceptance are separate gat ## Local regression boundary `python3 -I -B scripts/test_explicit_recovery.py` uses temporary repositories and -command doubles for authentication, installer, health and policy. It needs Linux +command doubles for authentication, installer, host collection and policy. The +rollback-state regressions use the production reconciliation-state reader and +health evaluator against persisted fixture records, including independent +critical host-health and state-persistence failures. The suite needs Linux Bash, Git, Python and flock, but no fleet credentials, Docker or live host. It proves orchestration and fail-closed routing, not actual image builds, installer transactions, systemd behavior or host acceptance. Complete exact-source/base diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 64f1ca3a..88b22498 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -94,9 +94,9 @@ require_commands() { save_reconcile_state() { local status=${1:-} desired_commit=${2:-} applied_commit=${3:-} health=${4:-} message=${5:-} mark_success=${6:-false} - local transaction_phase=${7:-} failure_reason=${8:-} + local transaction_phase=${7:-} failure_reason=${8:-} strict=${9:-false} write_status=0 install -d -m 0700 "$reconcile_state_dir" - python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || true + python3 - "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$mark_success" "$transaction_phase" "$failure_reason" <<'PY' 2>/dev/null || write_status=$? import json, os, sys, tempfile path = sys.argv[1] @@ -134,6 +134,31 @@ except: os.unlink(tmp, missing_ok=True) raise PY + [[ "$strict" == true ]] || return 0 + ((write_status == 0)) || return 1 + python3 -c ' +import json, sys +path, status, desired, applied, health, message, phase, reason = sys.argv[1:] +value = json.load(open(path, encoding="utf-8")) +expected = {"status": status, "desired_commit": desired, "applied_commit": applied, + "health": health, "message": message, "transaction_phase": phase, "failure_reason": reason} +if not isinstance(value, dict) or any(value.get(key) != item for key, item in expected.items()): + raise SystemExit(1) +' "$reconcile_state_file" "$status" "$desired_commit" "$applied_commit" "$health" "$message" "$transaction_phase" "$failure_reason" 2>/dev/null +} + +finalize_rollback_state() { + local health=$1 message=$2 + if save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$health" "$message" false '' '' true; then + return 0 + fi + if save_reconcile_state 'failed' "$desired_commit" "$restored_config_ref" "$health" 'terminal rollback state persistence rejected' false '' '' true; then + printf 'RECONCILE_ERROR: terminal rollback state persistence rejected; failure record verified\n' >&2 + else + printf 'STOP: reconciliation state persistence unverified; operator intervention required\n' >&2 + fi + # Never use die here: another unchecked write cannot establish the outcome. + return 1 } load_installed_state() { @@ -651,9 +676,12 @@ PY *) save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}; final health rejected" apply_lkg || die "final health was ${health_status}; rollback to last-known-good also failed" + save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good; checking rollback health' false '' '' true \ + || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown' rollback_health_status=$(run_health_check "$temp_dir/health.json") - verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified' - save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}" + verify_explicit_rollback_health "$rollback_health_status" \ + || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status" + finalize_rollback_state "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}" || exit 2 note "ROLLBACK_OK controller=${installed_controller} restored=${restored_config_ref}" exit 3 ;; @@ -685,10 +713,13 @@ finally: PY lkg_readback_matches "$restored_config_ref" || die 'baseline metadata restoration readback rejected' apply_lkg || die 'LKG persistence failed; drained baseline rollback failed' + save_reconcile_state 'reconciling' "$desired_commit" "$restored_config_ref" 'unknown' 'restored last-known-good after LKG persistence failure; checking rollback health' false '' '' true \ + || die 'rollback verification state persistence unverified' "$desired_commit" "$restored_config_ref" 'unknown' rollback_health_status=$(run_health_check "$temp_dir/health.json") - verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified' + verify_explicit_rollback_health "$rollback_health_status" \ + || die 'drained rollback health unverified' "$desired_commit" "$restored_config_ref" "$rollback_health_status" [[ "$rollback_health_status" == maintenance || "$rollback_health_status" == warning ]] || die 'LKG persistence failed; rollback health unverified' - save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" 'LKG persistence failed; restored drained baseline' + finalize_rollback_state "$rollback_health_status" 'LKG persistence failed; restored drained baseline' || exit 2 fi exit 3 fi diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py index c1532a5f..a8a87cdf 100644 --- a/scripts/test_explicit_recovery.py +++ b/scripts/test_explicit_recovery.py @@ -207,6 +207,238 @@ def test_pretransaction_denial_keeps_old_lkg(self): self.assertEqual(self._reconcile_state()["transaction_phase"], "pre_transaction") self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + def install_evaluated_health(self): + """Keep host collection doubled, but evaluate the live reconcile record.""" + scripts = self.root / "scripts" + for source, destination in (("health.py", "production_health.py"), + ("status_auth.py", "status_auth.py")): + shutil.copy2(Path(__file__).with_name(source), scripts / destination) + (scripts / "health.py").write_text('''import json, os, sys +from pathlib import Path +from production_health import Thresholds, evaluate, _reconcile_state + +calls = Path(os.environ["FAKE_INSTALLER_LOG"]).read_text().splitlines() +rollback = len(calls) > 1 +record = _reconcile_state(Path(os.environ["CI_FLEET_RECONCILE_STATE_DIR"]) / "state.json") +snapshot = { + "controller_id": "example-ci-01", "desired_state": "drained", + "disks": {"root": {"used_percent": 20, "inode_used_percent": 10}, + "docker": {"used_percent": 30, "inode_used_percent": 15}}, + "memory_available_percent": 75, "swap_used_percent": 0, "recent_oom": False, + "docker_available": rollback or os.environ.get("REJECT_CANDIDATE_HEALTH") != "1", + "controller": {"state": "missing", "restart_count": 0, "oom_killed": False}, + "configured_capacity": {"min": 0, "max": 1}, "effective_capacity": {"min": 0, "max": 0}, + "managed": {"running": 0, "inactive": 0, "unhealthy": 0, "restarting": 0}, + "stale": {"images": 0, "networks": 0, "volumes": 0, "build_cache_bytes": 0}, + "services": {"cleanup": "ok", "drift": "ok"}, + "timers": {"health": "ok", "cleanup": "ok", "drift": "ok", "updates": "ok"}, + "pending_reboot": False, "failed_packages": False, "clock_synchronized": True, + "backup": "not_configured", "reconciliation": record, +} +if rollback and os.environ.get("REJECT_ROLLBACK_HEALTH") == "1": + snapshot["docker_available"] = False +report = evaluate(snapshot, Thresholds()) +with open(os.environ["EVALUATED_HEALTH_LOG"], "a", encoding="utf-8") as handle: + handle.write(json.dumps({"rollback": rollback, "record": record, "report": report}) + "\\n") +Path(sys.argv[sys.argv.index("--output") + 1]).write_text(json.dumps(report), encoding="utf-8") +raise SystemExit(report["exit_code"]) +''', encoding="utf-8") + self.env["EVALUATED_HEALTH_LOG"] = str(self.td / "evaluated-health.jsonl") + directory = Path(self.env["CI_FLEET_RECONCILE_STATE_DIR"]) + directory.mkdir(mode=0o700, exist_ok=True) + (directory / "state.json").write_text(json.dumps({"last_success_at": 1})) + + def test_final_health_rollback_uses_production_reconciliation_evaluator(self): + self.install_evaluated_health() + self.env["REJECT_CANDIDATE_HEALTH"] = "1" + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr) + state = self._reconcile_state() + self.assertEqual(state["status"], "rolled_back") + self.assertEqual(state["applied_commit"], self.baseline) + self.assertEqual(state["last_success_at"], 1) + observations = [json.loads(line) for line in + Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()] + self.assertEqual(len(observations), 2) + self.assertTrue(observations[-1]["rollback"]) + self.assertEqual(observations[-1]["record"]["status"], "reconciling") + self.assertEqual(observations[-1]["record"]["applied_commit"], self.baseline) + self.assertEqual(observations[-1]["report"]["status"], "warning") + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_lkg_failure_rollback_uses_production_reconciliation_evaluator(self): + for failure in ("write", "missing_write", "readback"): + with self.subTest(failure=failure): + self.setUp() + self.install_evaluated_health() + self.env["DENY_LKG_REF"] = self.candidate + real_python = shutil.which("python3") + assert real_python is not None + write_outcome = "exit 1" if failure == "write" else "exit 0" + if failure == "readback": + condition = '${1:-} == -c && ${3:-} == "$CI_FLEET_LKG_DIR/metadata.json" && ${5:-} == "$DENY_LKG_REF"' + write_outcome = "exit 1" + else: + condition = '${1:-} == - && ${2:-} == "$CI_FLEET_LKG_DIR/metadata.json" && ${4:-} == "$DENY_LKG_REF"' + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ {condition} ]]; then {write_outcome}; fi +exec {shlex.quote(real_python)} "$@" +''') + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + state = self._reconcile_state() + self.assertEqual(state["status"], "rolled_back", result.stdout + result.stderr) + self.assertEqual(state["applied_commit"], self.baseline) + self.assertEqual(state["last_success_at"], 1) + self.assertNotIn("RECONCILE_OK", result.stdout) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + observations = [json.loads(line) for line in + Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()] + self.assertTrue(observations[-1]["rollback"]) + self.assertEqual(observations[-1]["record"]["status"], "reconciling") + self.assertEqual(observations[-1]["record"]["applied_commit"], self.baseline) + self.assertEqual(observations[-1]["report"]["status"], "warning") + + def test_missing_rollback_state_persistence_cannot_verify_recovery(self): + self.install_evaluated_health() + self.env["REJECT_CANDIDATE_HEALTH"] = "1" + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == reconciling ]] && \ + [[ $(wc -l < "$FAKE_INSTALLER_LOG") -gt 1 ]]; then + rm -f -- "$CI_FLEET_RECONCILE_STATE_DIR/state.json" + exit 1 +fi +exec {shlex.quote(real_python)} "$@" +''') + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + + def test_stale_rollback_state_readback_cannot_verify_recovery(self): + self.install_evaluated_health() + self.env["REJECT_CANDIDATE_HEALTH"] = "1" + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == reconciling ]] && \ + [[ $(wc -l < "$FAKE_INSTALLER_LOG") -gt 1 ]]; then + rm -f -- "$CI_FLEET_RECONCILE_STATE_DIR/state.json" + exit 0 +fi +exec {shlex.quote(real_python)} "$@" +''') + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + + def test_production_rollback_health_still_rejects_independent_failure(self): + self.install_evaluated_health() + self.env.update(REJECT_CANDIDATE_HEALTH="1", REJECT_ROLLBACK_HEALTH="1") + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("ROLLBACK_OK", result.stdout) + record = self._reconcile_state() + self.assertEqual(record["status"], "failed") + self.assertEqual(record["applied_commit"], self.baseline) + self.assertEqual(record["last_success_at"], 1) + observations = [json.loads(line) for line in + Path(self.env["EVALUATED_HEALTH_LOG"]).read_text().splitlines()] + self.assertTrue(observations[-1]["rollback"]) + self.assertEqual(observations[-1]["report"]["status"], "unhealthy") + self.assertIn("docker", {check["id"] for check in observations[-1]["report"]["checks"] + if check["status"] == "critical"}) + + def assert_terminal_rollback_failure_is_recorded(self, *, lkg_failure): + for failure in ("write", "missing_write", "readback"): + with self.subTest(failure=failure, lkg_failure=lkg_failure): + self.setUp() + self.install_evaluated_health() + if not lkg_failure: + self.env["REJECT_CANDIDATE_HEALTH"] = "1" + self.env.update(TERMINAL_FAILURE=failure, + DENY_LKG_REF=self.candidate if lkg_failure else "") + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ -n "$DENY_LKG_REF" && ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then + exit 1 +fi +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{3:-}} == rolled_back ]]; then + case "$TERMINAL_FAILURE" in + write) exit 1 ;; + missing_write) exit 0 ;; + esac +fi +if [[ ${{1:-}} == -c && ${{3:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" && ${{4:-}} == rolled_back && "$TERMINAL_FAILURE" == readback ]]; then + exit 1 +fi +exec {shlex.quote(real_python)} "$@" +''') + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("ROLLBACK_OK", result.stdout, result.stdout + result.stderr) + self.assertNotIn("RECONCILE_OK", result.stdout) + record = self._reconcile_state() + self.assertEqual(record["status"], "failed", result.stdout + result.stderr) + self.assertEqual(record["desired_commit"], self.candidate) + self.assertEqual(record["applied_commit"], self.baseline) + self.assertEqual(record["last_success_at"], 1) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + report = self.td / "ordinary-health.json" + ordinary = subprocess.run([real_python, str(self.root / "scripts/health.py"), + "--output", str(report)], env=self.env, + capture_output=True, text=True, timeout=10) + self.assertEqual(ordinary.returncode, 2, ordinary.stdout + ordinary.stderr) + self.assertEqual(json.loads(report.read_text())["status"], "unhealthy") + + def test_final_health_terminal_persistence_failure_stays_critical(self): + self.assert_terminal_rollback_failure_is_recorded(lkg_failure=False) + + def test_lkg_terminal_persistence_failure_stays_critical(self): + self.assert_terminal_rollback_failure_is_recorded(lkg_failure=True) + + def test_terminal_and_fallback_persistence_failure_requires_operator(self): + for lkg_failure in (False, True): + with self.subTest(lkg_failure=lkg_failure): + self.setUp() + self.install_evaluated_health() + if not lkg_failure: + self.env["REJECT_CANDIDATE_HEALTH"] = "1" + self.env.update(DENY_LKG_REF=self.candidate if lkg_failure else "", + TERMINAL_ATTEMPT=str(self.td / "terminal-attempt")) + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ -n "$DENY_LKG_REF" && ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then + exit 1 +fi +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_RECONCILE_STATE_DIR/state.json" ]]; then + if [[ ${{3:-}} == rolled_back ]]; then + : > "$TERMINAL_ATTEMPT" + exit 1 + fi + if [[ ${{3:-}} == failed && -f "$TERMINAL_ATTEMPT" ]]; then exit 1; fi +fi +exec {shlex.quote(real_python)} "$@" +''') + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("STOP: reconciliation state persistence unverified; operator intervention required", + result.stderr) + self.assertNotIn("ROLLBACK_OK", result.stdout) + self.assertNotIn("RECONCILE_OK", result.stdout) + record = self._reconcile_state() + self.assertEqual(record["status"], "reconciling") + self.assertEqual(record["desired_commit"], self.candidate) + self.assertEqual(record["applied_commit"], self.baseline) + self.assertEqual(record["last_success_at"], 1) + self.assertTrue(Path(self.env["TERMINAL_ATTEMPT"]).is_file()) + def test_final_health_failure_rolls_back_to_drained_baseline(self): self.env["FAKE_HEALTH_STATUS"] = "unhealthy" result = self.run_ref("--ref", self.candidate)