From 96432fd23b115bbe07cf94660ba7c669b17b76f4 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 31 Oct 2026 04:26:53 -0500 Subject: [PATCH 1/2] fix: accept a verified installed recovery baseline --- docs/EXPLICIT-DRAINED-RECOVERY.md | 36 +++++++++ scripts/remote-reconcile.sh | 40 +++++++++- scripts/test_remote_reconcile.py | 123 +++++++++++++++++++++++++++++- 3 files changed, 196 insertions(+), 3 deletions(-) diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md index 56c24bbf..e17da9bb 100644 --- a/docs/EXPLICIT-DRAINED-RECOVERY.md +++ b/docs/EXPLICIT-DRAINED-RECOVERY.md @@ -86,6 +86,42 @@ nonzero exit. Failed recovery must remain held; admission needs separate review and authorization. This command does not establish active-pilot evidence, isolation proof or permission to publish otherwise unapproved images. +## Accept an already installed recovery baseline + +A supporting reviewed reconciliation checkout can accept the current installed +configuration as last-known-good without another installer transaction: + +```bash +sudo /path/to/reviewed-ci-fleet/scripts/remote-reconcile.sh --accept-installed +``` + +Use the exact independently reviewed, merged source that supports this command. +Older managers do not support it. The runtime and installed manager do not need +to be replaced for this verification. Keep the reviewed capacities, workload +protections and automation holds in place throughout acceptance. + +The command holds the installer lock and fetches only the locked installation's +full configuration identity. It requires the existing approval policy, an exact +installed check and fresh state-aware full health before replacing LKG through +the normal checked metadata writer. Healthy or warning is accepted for active +controllers; drained and disabled controllers require matching maintenance or +warning health. The prior record remains in place during verification. Save its +protected metadata and recovery-source evidence outside Git before acceptance. + +Drift, a mismatched fetch, missing or inconsistent health, and metadata write or +readback failure stop acceptance. This command never repairs configuration or +applies the older LKG, including on failure. It cannot be combined with another +reconciliation mode. Normal same-commit reconciliation, `--check-only`, +`--installed-ref` and `--no-op` retain their existing behavior and do not accept an +installed LKG. Inspect the terminal reconciliation record after any failure and +keep automation held until the actual result is established. + +This provides a separate route when an older LKG has an incompatible capacity or +resource policy. It does not authorize restoration of that older state, changes +to a checkpoint, an engine upgrade, workload retirement or storage cleanup. +Template guidance and private adopter verification must follow the accepted +supporting source before live use. + ## Rollout boundary This feature requires a new reviewed supporting engine/manager pin and complete diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 0761c511..047a18eb 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -7,6 +7,7 @@ # # Usage: # remote-reconcile.sh [--check-only] [--installed-ref] [--no-op] +# remote-reconcile.sh --accept-installed set -Eeuo pipefail set +x export PYTHONDONTWRITEBYTECODE=1 @@ -29,6 +30,7 @@ lkg_dir=${CI_FLEET_LKG_DIR:-/var/lib/ci-fleet/last-known-good} mode=reconcile # reconcile or check-only no_op=false installed_ref=false +accept_installed=false explicit_ref= max_attempts=${CI_FLEET_RECONCILE_MAX_ATTEMPTS:-3} # ponytail: one-hour queue cap; align with any future installer transaction timeout. @@ -38,15 +40,19 @@ usage() { cat >&2 <<'EOF' usage: remote-reconcile.sh [--check-only] [--installed-ref] [--no-op] + remote-reconcile.sh --accept-installed Fetches the desired-state repository at the current default-branch HEAD, validates it, and reconciles the controller if a newer commit is available. --ref SHA Mutating opt-in drained recovery at one full lowercase 40-hex commit; unchanged controller/fleet policy and runtime engine only. - Incompatible with --check-only, --installed-ref and --no-op. + Incompatible with --check-only, --installed-ref, --no-op and --accept-installed. --check-only Validate and report without reconciling. --installed-ref Check the locked installation's commit (check-only only). + --accept-installed Verify only the locked installed commit and full health, + then accept it as LKG without applying or repairing configuration. + Incompatible with --check-only, --installed-ref, --no-op and --ref. --no-op Log what would be done without side effects. EOF } @@ -58,6 +64,9 @@ while (($#)); do explicit_ref=$2; shift ;; --check-only) mode=check-only ;; --installed-ref) installed_ref=true ;; + --accept-installed) + [[ "$accept_installed" == false ]] || { echo 'ERROR: duplicate --accept-installed' >&2; exit 2; } + accept_installed=true ;; --no-op) no_op=true ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: unknown argument: $1" >&2; usage; exit 2 ;; @@ -66,6 +75,7 @@ while (($#)); do done [[ -z "$explicit_ref" || ( "$installed_ref" == false && "$no_op" == false && "$mode" == reconcile ) ]] || { echo 'ERROR: --ref is incompatible with --installed-ref, --check-only and --no-op' >&2; exit 2; } [[ "$installed_ref" == false || "$mode" == check-only ]] || { echo 'ERROR: --installed-ref requires --check-only' >&2; exit 2; } +[[ "$accept_installed" == false || ( "$mode" == reconcile && "$installed_ref" == false && "$no_op" == false && -z "$explicit_ref" ) ]] || { echo 'ERROR: --accept-installed cannot be combined with another mode' >&2; exit 2; } temp_dir=$(mktemp -d) || exit 2 transaction_result=$temp_dir/transaction-result.json @@ -517,6 +527,7 @@ if [[ -n "$explicit_ref" ]]; then fetch_ref=$explicit_ref fi [[ "$installed_ref" == false ]] || fetch_ref=$installed_config_ref +[[ "$accept_installed" == false ]] || fetch_ref=$installed_config_ref note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}" # Prefer host.env for token generation; fall back to rendered_env @@ -554,6 +565,7 @@ while ((attempt < max_attempts)); do done note "RESOLVED desired_ref=${desired_commit}" +[[ "$accept_installed" == false || "$desired_commit" == "$installed_config_ref" ]] || die 'installed acceptance fetch identity mismatch' if [[ -n "$explicit_ref" ]]; then [[ "$desired_commit" == "$explicit_ref" ]] || die 'explicit ref fetch identity mismatch' git -C "$temp_dir/config-repo" show "$desired_commit:fleet.json" >"$temp_dir/recovery-candidate.json" || die 'candidate unavailable' @@ -590,11 +602,35 @@ if [[ "$desired_commit" == "$installed_config_ref" && -z "$explicit_ref" ]]; the --config-identity "$installed_config_repo" \ --ref "$installed_config_ref" \ --controller "$installed_controller" 2>"$temp_dir/drift_err"; then + health_status=healthy + if [[ "$accept_installed" == true ]]; then + save_reconcile_state 'reconciling' "$desired_commit" "$installed_config_ref" 'unknown' 'installed configuration checked; verifying health' false '' '' true \ + || die 'installed acceptance state persistence unverified' + health_status=$(run_health_check "$temp_dir/health.json") + health_desired_state=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["desired_state"])' "$temp_dir/health.json" 2>/dev/null || echo unknown) + case "$installed_controller_state:$health_desired_state:$health_status" in + active:active:healthy|active:active:warning|drained:drained:maintenance|drained:drained:warning|disabled:disabled:maintenance|disabled:disabled:warning) ;; + *) + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" "$health_status" 'installed configuration health rejected' + exit 3 + ;; + esac + # A failed verification must never replay an older, potentially incompatible LKG. + if ! { save_lkg "$installed_config_ref" true && lkg_readback_matches "$installed_config_ref"; }; then + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" "$health_status" 'installed LKG persistence failed' + exit 3 + fi + fi + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" "$health_status" 'no change, converged' true '' '' true \ + || die 'installed convergence state persistence unverified' note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}" - save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' true exit 0 fi fi + if [[ "$accept_installed" == true ]]; then + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" 'unknown' 'installed configuration check rejected; no repair attempted' + exit 3 + fi # Same commit + drift is still drift: check-only reports it, normal mode repairs it. if [[ "$mode" == check-only ]]; then save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected' diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py index 4ea6b84c..077efb70 100644 --- a/scripts/test_remote_reconcile.py +++ b/scripts/test_remote_reconcile.py @@ -11,6 +11,7 @@ import shlex import shutil import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -552,6 +553,7 @@ def _run( policy_state: str = "healthy", transaction_record: str = "", transaction_exit: str = "2", + arguments: tuple[str, ...] = (), ): env = dict(self.env) env.update( @@ -565,7 +567,7 @@ def _run( FAKE_TRANSACTION_EXIT=transaction_exit, ) return subprocess.run( - [str(self.root / "scripts" / "remote-reconcile.sh")], + [str(self.root / "scripts" / "remote-reconcile.sh"), *arguments], capture_output=True, text=True, env=env, @@ -582,6 +584,125 @@ def _reconcile_state(self) -> dict: path = Path(self.env["CI_FLEET_RECONCILE_STATE_DIR"]) / "state.json" return json.loads(path.read_text(encoding="utf-8")) + def _installed_candidate_with_retained_lkg(self, desired_state="active") -> bytes: + self._reset(self.prior_ref, desired_state) + self._write_installed_ref(self.desired_ref, desired_state) + path = self.lkg_dir / "metadata.json" + value = json.loads(path.read_text(encoding="utf-8")) + value["saved_at"] = 1 + path.write_text(json.dumps(value), encoding="utf-8") + path.chmod(0o600) + return path.read_bytes() + + def test_same_commit_accepts_installed_lkg_after_state_aware_health(self): + for desired_state, health in (("active", "healthy"), ("active", "warning"), + ("drained", "maintenance"), ("drained", "warning"), + ("disabled", "maintenance"), ("disabled", "warning")): + with self.subTest(desired_state=desired_state, health=health): + self._installed_candidate_with_retained_lkg(desired_state) + result = self._run(health, desired_state=desired_state, arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual(self.health_observed_lkg.read_text().strip(), self.prior_ref) + self.assertEqual(self._lkg_ref(), self.desired_ref) + state = self._reconcile_state() + self.assertEqual(state["status"], "converged") + self.assertEqual(state["health"], health) + + def test_same_commit_unaccepted_health_preserves_lkg_without_rollback(self): + for health, reported_state in (("unhealthy", "active"), ("malformed", "active"), + ("maintenance", "active"), ("healthy", "drained")): + with self.subTest(health=health, reported_state=reported_state): + original = self._installed_candidate_with_retained_lkg() + installed = self.state_file.read_bytes() + result = self._run(health, desired_state=reported_state, arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertEqual(self.state_file.read_bytes(), installed) + self.assertEqual(self._reconcile_state()["status"], "failed") + self.assertNotIn("ROLLBACK_OK", result.stdout) + self.assertNotIn("CONVERGED controller=", result.stdout) + + def test_same_commit_readonly_modes_do_not_accept_lkg(self): + for argument in ("--check-only", "--no-op", None): + with self.subTest(argument=argument): + original = self._installed_candidate_with_retained_lkg() + result = self._run("healthy", arguments=(argument,) if argument else ()) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertFalse(self.health_observed_lkg.exists()) + if argument != "--no-op": + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + else: + self.assertFalse(self.installer_log.exists()) + + def test_accept_installed_rejects_mode_combinations_before_authentication(self): + for other in (("--check-only",), ("--installed-ref",), ("--no-op",), + ("--ref", "a" * 40), ("--accept-installed",)): + for arguments in (("--accept-installed", *other), (*other, "--accept-installed")): + with self.subTest(arguments=arguments): + original = self._installed_candidate_with_retained_lkg() + result = self._run("healthy", arguments=arguments) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertNotIn("GENERATING_TOKEN", result.stdout) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + + def test_accept_installed_never_repairs_failed_installed_check(self): + original = self._installed_candidate_with_retained_lkg() + installed = self.state_file.read_bytes() + result = self._run("healthy", policy_state="corrupt", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertEqual(self.state_file.read_bytes(), installed) + self.assertFalse(self.health_observed_lkg.exists()) + + def test_accept_installed_requires_lkg_persistence_and_readback(self): + for failure in ("write", "readback"): + with self.subTest(failure=failure): + if (self.lkg_dir / "metadata.json").is_dir(): + (self.lkg_dir / "metadata.json").rmdir() + self._installed_candidate_with_retained_lkg() + if failure == "write": + path = self.lkg_dir / "metadata.json" + path.unlink() + path.mkdir() + else: + self._write_executable(self.td / "bin" / "python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == -c && ${{3:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{5:-}} == "$REMOTE_FIXTURE_REF" ]]; then exit 1; fi +exec {shlex.quote(sys.executable)} "$@" +''') + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual(self._reconcile_state()["status"], "failed") + self.assertNotIn("CONVERGED controller=", result.stdout) + + def test_accept_installed_fetches_only_installed_ref_and_rejects_wrong_result(self): + self._installed_candidate_with_retained_lkg() + fetch_log = self.td / "fetches" + wrapper = self.td / "bin" / "git" + wrapper.write_text(wrapper.read_text().replace( + 'if [[ ${1:-} == -C && ${3:-} == fetch ]]; then', + 'if [[ ${1:-} == -C && ${3:-} == fetch ]]; then\n' + + ' printf "%s\\n" "${@: -1}" >>' + shlex.quote(str(fetch_log))), encoding="utf-8") + self.assertEqual(git("commit", "--allow-empty", "-qm", "later default branch", cwd=self.config_repo).returncode, 0) + newer_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(fetch_log.read_text().splitlines(), [self.desired_ref]) + self.assertEqual(self._lkg_ref(), self.desired_ref) + + original = self._installed_candidate_with_retained_lkg() + self.env["REMOTE_FIXTURE_REF"] = newer_ref + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("installed acceptance fetch identity mismatch", result.stderr) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + def _assert_rolled_back(self, result: subprocess.CompletedProcess, health: str) -> None: self.assertEqual(result.returncode, 3, result.stderr) self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--upgrade", "--upgrade"]) From 49b43ae0b4fcc901a0b9766baf832eb80a431356 Mon Sep 17 00:00:00 2001 From: Nick Foster Date: Sat, 31 Oct 2026 05:35:58 -0500 Subject: [PATCH 2/2] fix: reject unactivated installed recovery baselines --- docs/EXPLICIT-DRAINED-RECOVERY.md | 4 +++- scripts/remote-reconcile.sh | 11 +++++++++++ scripts/test_remote_reconcile.py | 26 ++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md index e17da9bb..3b158aff 100644 --- a/docs/EXPLICIT-DRAINED-RECOVERY.md +++ b/docs/EXPLICIT-DRAINED-RECOVERY.md @@ -103,7 +103,9 @@ protections and automation holds in place throughout acceptance. The command holds the installer lock and fetches only the locked installation's full configuration identity. It requires the existing approval policy, an exact installed check and fresh state-aware full health before replacing LKG through -the normal checked metadata writer. Healthy or warning is accepted for active +the normal checked metadata writer. Authored managed-image policy is required; +image-free or staged baselines need their separate protected-checkpoint recovery +procedure and are refused by this command. Healthy or warning is accepted for active controllers; drained and disabled controllers require matching maintenance or warning health. The prior record remains in place during verification. Save its protected metadata and recovery-source evidence outside Git before acceptance. diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 047a18eb..ac48160f 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -584,6 +584,17 @@ if [[ -e "$config_review_policy" || -L "$config_review_policy" || -e "$config_re fi fi +if [[ "$accept_installed" == true ]]; then + git -C "$temp_dir/config-repo" show "$desired_commit:fleet.json" >"$temp_dir/acceptance-fleet.json" \ + || die 'installed acceptance configuration is unavailable' + python3 - "$temp_dir/acceptance-fleet.json" <<'PY' || die 'installed acceptance requires an activated managed-image configuration' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +images = value.get("managed_images") if isinstance(value, dict) else None +raise SystemExit(0 if isinstance(images, dict) and images else 1) +PY +fi + # Compare with installed if [[ "$desired_commit" == "$installed_config_ref" && -z "$explicit_ref" ]]; then # Same commit — just verify convergence diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py index 077efb70..b31004fb 100644 --- a/scripts/test_remote_reconcile.py +++ b/scripts/test_remote_reconcile.py @@ -585,6 +585,15 @@ def _reconcile_state(self) -> dict: return json.loads(path.read_text(encoding="utf-8")) def _installed_candidate_with_retained_lkg(self, desired_state="active") -> bytes: + fleet = json.loads((self.config_repo / "fleet.json").read_text()) + if not fleet.get("managed_images"): + fleet["managed_images"] = {"1" * 40: {"amd64": { + "controller": "sha256:" + "a" * 64, "runner": "sha256:" + "b" * 64}}} + (self.config_repo / "fleet.json").write_text(json.dumps(fleet), encoding="utf-8") + self.assertEqual(git("add", "fleet.json", cwd=self.config_repo).returncode, 0) + self.assertEqual(git("commit", "-qm", "activated fixture", cwd=self.config_repo).returncode, 0) + self.desired_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + self.env["REMOTE_FIXTURE_REF"] = self.desired_ref self._reset(self.prior_ref, desired_state) self._write_installed_ref(self.desired_ref, desired_state) path = self.lkg_dir / "metadata.json" @@ -659,6 +668,23 @@ def test_accept_installed_never_repairs_failed_installed_check(self): self.assertEqual(self.state_file.read_bytes(), installed) self.assertFalse(self.health_observed_lkg.exists()) + def test_accept_installed_rejects_image_free_or_staged_configuration(self): + for images in (None, {}, []): + with self.subTest(images=images): + original = self._installed_candidate_with_retained_lkg() + fleet = {} if images is None else {"managed_images": images} + (self.config_repo / "fleet.json").write_text(json.dumps(fleet), encoding="utf-8") + self.assertEqual(git("add", "fleet.json", cwd=self.config_repo).returncode, 0) + self.assertEqual(git("commit", "--allow-empty", "-qm", "unactivated fixture", cwd=self.config_repo).returncode, 0) + self.desired_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + self.env["REMOTE_FIXTURE_REF"] = self.desired_ref + self._write_installed_ref(self.desired_ref) + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("activated managed-image configuration", result.stderr) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + def test_accept_installed_requires_lkg_persistence_and_readback(self): for failure in ("write", "readback"): with self.subTest(failure=failure):