diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md index 56c24bbf..3b158aff 100644 --- a/docs/EXPLICIT-DRAINED-RECOVERY.md +++ b/docs/EXPLICIT-DRAINED-RECOVERY.md @@ -86,6 +86,44 @@ nonzero exit. Failed recovery must remain held; admission needs separate review and authorization. This command does not establish active-pilot evidence, isolation proof or permission to publish otherwise unapproved images. +## Accept an already installed recovery baseline + +A supporting reviewed reconciliation checkout can accept the current installed +configuration as last-known-good without another installer transaction: + +```bash +sudo /path/to/reviewed-ci-fleet/scripts/remote-reconcile.sh --accept-installed +``` + +Use the exact independently reviewed, merged source that supports this command. +Older managers do not support it. The runtime and installed manager do not need +to be replaced for this verification. Keep the reviewed capacities, workload +protections and automation holds in place throughout acceptance. + +The command holds the installer lock and fetches only the locked installation's +full configuration identity. It requires the existing approval policy, an exact +installed check and fresh state-aware full health before replacing LKG through +the normal checked metadata writer. Authored managed-image policy is required; +image-free or staged baselines need their separate protected-checkpoint recovery +procedure and are refused by this command. Healthy or warning is accepted for active +controllers; drained and disabled controllers require matching maintenance or +warning health. The prior record remains in place during verification. Save its +protected metadata and recovery-source evidence outside Git before acceptance. + +Drift, a mismatched fetch, missing or inconsistent health, and metadata write or +readback failure stop acceptance. This command never repairs configuration or +applies the older LKG, including on failure. It cannot be combined with another +reconciliation mode. Normal same-commit reconciliation, `--check-only`, +`--installed-ref` and `--no-op` retain their existing behavior and do not accept an +installed LKG. Inspect the terminal reconciliation record after any failure and +keep automation held until the actual result is established. + +This provides a separate route when an older LKG has an incompatible capacity or +resource policy. It does not authorize restoration of that older state, changes +to a checkpoint, an engine upgrade, workload retirement or storage cleanup. +Template guidance and private adopter verification must follow the accepted +supporting source before live use. + ## Rollout boundary This feature requires a new reviewed supporting engine/manager pin and complete diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 0761c511..ac48160f 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -7,6 +7,7 @@ # # Usage: # remote-reconcile.sh [--check-only] [--installed-ref] [--no-op] +# remote-reconcile.sh --accept-installed set -Eeuo pipefail set +x export PYTHONDONTWRITEBYTECODE=1 @@ -29,6 +30,7 @@ lkg_dir=${CI_FLEET_LKG_DIR:-/var/lib/ci-fleet/last-known-good} mode=reconcile # reconcile or check-only no_op=false installed_ref=false +accept_installed=false explicit_ref= max_attempts=${CI_FLEET_RECONCILE_MAX_ATTEMPTS:-3} # ponytail: one-hour queue cap; align with any future installer transaction timeout. @@ -38,15 +40,19 @@ usage() { cat >&2 <<'EOF' usage: remote-reconcile.sh [--check-only] [--installed-ref] [--no-op] + remote-reconcile.sh --accept-installed Fetches the desired-state repository at the current default-branch HEAD, validates it, and reconciles the controller if a newer commit is available. --ref SHA Mutating opt-in drained recovery at one full lowercase 40-hex commit; unchanged controller/fleet policy and runtime engine only. - Incompatible with --check-only, --installed-ref and --no-op. + Incompatible with --check-only, --installed-ref, --no-op and --accept-installed. --check-only Validate and report without reconciling. --installed-ref Check the locked installation's commit (check-only only). + --accept-installed Verify only the locked installed commit and full health, + then accept it as LKG without applying or repairing configuration. + Incompatible with --check-only, --installed-ref, --no-op and --ref. --no-op Log what would be done without side effects. EOF } @@ -58,6 +64,9 @@ while (($#)); do explicit_ref=$2; shift ;; --check-only) mode=check-only ;; --installed-ref) installed_ref=true ;; + --accept-installed) + [[ "$accept_installed" == false ]] || { echo 'ERROR: duplicate --accept-installed' >&2; exit 2; } + accept_installed=true ;; --no-op) no_op=true ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: unknown argument: $1" >&2; usage; exit 2 ;; @@ -66,6 +75,7 @@ while (($#)); do done [[ -z "$explicit_ref" || ( "$installed_ref" == false && "$no_op" == false && "$mode" == reconcile ) ]] || { echo 'ERROR: --ref is incompatible with --installed-ref, --check-only and --no-op' >&2; exit 2; } [[ "$installed_ref" == false || "$mode" == check-only ]] || { echo 'ERROR: --installed-ref requires --check-only' >&2; exit 2; } +[[ "$accept_installed" == false || ( "$mode" == reconcile && "$installed_ref" == false && "$no_op" == false && -z "$explicit_ref" ) ]] || { echo 'ERROR: --accept-installed cannot be combined with another mode' >&2; exit 2; } temp_dir=$(mktemp -d) || exit 2 transaction_result=$temp_dir/transaction-result.json @@ -517,6 +527,7 @@ if [[ -n "$explicit_ref" ]]; then fetch_ref=$explicit_ref fi [[ "$installed_ref" == false ]] || fetch_ref=$installed_config_ref +[[ "$accept_installed" == false ]] || fetch_ref=$installed_config_ref note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}" # Prefer host.env for token generation; fall back to rendered_env @@ -554,6 +565,7 @@ while ((attempt < max_attempts)); do done note "RESOLVED desired_ref=${desired_commit}" +[[ "$accept_installed" == false || "$desired_commit" == "$installed_config_ref" ]] || die 'installed acceptance fetch identity mismatch' if [[ -n "$explicit_ref" ]]; then [[ "$desired_commit" == "$explicit_ref" ]] || die 'explicit ref fetch identity mismatch' git -C "$temp_dir/config-repo" show "$desired_commit:fleet.json" >"$temp_dir/recovery-candidate.json" || die 'candidate unavailable' @@ -572,6 +584,17 @@ if [[ -e "$config_review_policy" || -L "$config_review_policy" || -e "$config_re fi fi +if [[ "$accept_installed" == true ]]; then + git -C "$temp_dir/config-repo" show "$desired_commit:fleet.json" >"$temp_dir/acceptance-fleet.json" \ + || die 'installed acceptance configuration is unavailable' + python3 - "$temp_dir/acceptance-fleet.json" <<'PY' || die 'installed acceptance requires an activated managed-image configuration' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +images = value.get("managed_images") if isinstance(value, dict) else None +raise SystemExit(0 if isinstance(images, dict) and images else 1) +PY +fi + # Compare with installed if [[ "$desired_commit" == "$installed_config_ref" && -z "$explicit_ref" ]]; then # Same commit — just verify convergence @@ -590,11 +613,35 @@ if [[ "$desired_commit" == "$installed_config_ref" && -z "$explicit_ref" ]]; the --config-identity "$installed_config_repo" \ --ref "$installed_config_ref" \ --controller "$installed_controller" 2>"$temp_dir/drift_err"; then + health_status=healthy + if [[ "$accept_installed" == true ]]; then + save_reconcile_state 'reconciling' "$desired_commit" "$installed_config_ref" 'unknown' 'installed configuration checked; verifying health' false '' '' true \ + || die 'installed acceptance state persistence unverified' + health_status=$(run_health_check "$temp_dir/health.json") + health_desired_state=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["desired_state"])' "$temp_dir/health.json" 2>/dev/null || echo unknown) + case "$installed_controller_state:$health_desired_state:$health_status" in + active:active:healthy|active:active:warning|drained:drained:maintenance|drained:drained:warning|disabled:disabled:maintenance|disabled:disabled:warning) ;; + *) + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" "$health_status" 'installed configuration health rejected' + exit 3 + ;; + esac + # A failed verification must never replay an older, potentially incompatible LKG. + if ! { save_lkg "$installed_config_ref" true && lkg_readback_matches "$installed_config_ref"; }; then + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" "$health_status" 'installed LKG persistence failed' + exit 3 + fi + fi + save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" "$health_status" 'no change, converged' true '' '' true \ + || die 'installed convergence state persistence unverified' note "CONVERGED controller=${installed_controller} config_ref=${installed_config_ref}" - save_reconcile_state 'converged' "$desired_commit" "$installed_config_ref" 'healthy' 'no change, converged' true exit 0 fi fi + if [[ "$accept_installed" == true ]]; then + save_reconcile_state 'failed' "$desired_commit" "$installed_config_ref" 'unknown' 'installed configuration check rejected; no repair attempted' + exit 3 + fi # Same commit + drift is still drift: check-only reports it, normal mode repairs it. if [[ "$mode" == check-only ]]; then save_reconcile_state 'drift' "$desired_commit" "$installed_config_ref" 'drift' 'internal drift detected' diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py index 4ea6b84c..b31004fb 100644 --- a/scripts/test_remote_reconcile.py +++ b/scripts/test_remote_reconcile.py @@ -11,6 +11,7 @@ import shlex import shutil import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -552,6 +553,7 @@ def _run( policy_state: str = "healthy", transaction_record: str = "", transaction_exit: str = "2", + arguments: tuple[str, ...] = (), ): env = dict(self.env) env.update( @@ -565,7 +567,7 @@ def _run( FAKE_TRANSACTION_EXIT=transaction_exit, ) return subprocess.run( - [str(self.root / "scripts" / "remote-reconcile.sh")], + [str(self.root / "scripts" / "remote-reconcile.sh"), *arguments], capture_output=True, text=True, env=env, @@ -582,6 +584,151 @@ def _reconcile_state(self) -> dict: path = Path(self.env["CI_FLEET_RECONCILE_STATE_DIR"]) / "state.json" return json.loads(path.read_text(encoding="utf-8")) + def _installed_candidate_with_retained_lkg(self, desired_state="active") -> bytes: + fleet = json.loads((self.config_repo / "fleet.json").read_text()) + if not fleet.get("managed_images"): + fleet["managed_images"] = {"1" * 40: {"amd64": { + "controller": "sha256:" + "a" * 64, "runner": "sha256:" + "b" * 64}}} + (self.config_repo / "fleet.json").write_text(json.dumps(fleet), encoding="utf-8") + self.assertEqual(git("add", "fleet.json", cwd=self.config_repo).returncode, 0) + self.assertEqual(git("commit", "-qm", "activated fixture", cwd=self.config_repo).returncode, 0) + self.desired_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + self.env["REMOTE_FIXTURE_REF"] = self.desired_ref + self._reset(self.prior_ref, desired_state) + self._write_installed_ref(self.desired_ref, desired_state) + path = self.lkg_dir / "metadata.json" + value = json.loads(path.read_text(encoding="utf-8")) + value["saved_at"] = 1 + path.write_text(json.dumps(value), encoding="utf-8") + path.chmod(0o600) + return path.read_bytes() + + def test_same_commit_accepts_installed_lkg_after_state_aware_health(self): + for desired_state, health in (("active", "healthy"), ("active", "warning"), + ("drained", "maintenance"), ("drained", "warning"), + ("disabled", "maintenance"), ("disabled", "warning")): + with self.subTest(desired_state=desired_state, health=health): + self._installed_candidate_with_retained_lkg(desired_state) + result = self._run(health, desired_state=desired_state, arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual(self.health_observed_lkg.read_text().strip(), self.prior_ref) + self.assertEqual(self._lkg_ref(), self.desired_ref) + state = self._reconcile_state() + self.assertEqual(state["status"], "converged") + self.assertEqual(state["health"], health) + + def test_same_commit_unaccepted_health_preserves_lkg_without_rollback(self): + for health, reported_state in (("unhealthy", "active"), ("malformed", "active"), + ("maintenance", "active"), ("healthy", "drained")): + with self.subTest(health=health, reported_state=reported_state): + original = self._installed_candidate_with_retained_lkg() + installed = self.state_file.read_bytes() + result = self._run(health, desired_state=reported_state, arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertEqual(self.state_file.read_bytes(), installed) + self.assertEqual(self._reconcile_state()["status"], "failed") + self.assertNotIn("ROLLBACK_OK", result.stdout) + self.assertNotIn("CONVERGED controller=", result.stdout) + + def test_same_commit_readonly_modes_do_not_accept_lkg(self): + for argument in ("--check-only", "--no-op", None): + with self.subTest(argument=argument): + original = self._installed_candidate_with_retained_lkg() + result = self._run("healthy", arguments=(argument,) if argument else ()) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertFalse(self.health_observed_lkg.exists()) + if argument != "--no-op": + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + else: + self.assertFalse(self.installer_log.exists()) + + def test_accept_installed_rejects_mode_combinations_before_authentication(self): + for other in (("--check-only",), ("--installed-ref",), ("--no-op",), + ("--ref", "a" * 40), ("--accept-installed",)): + for arguments in (("--accept-installed", *other), (*other, "--accept-installed")): + with self.subTest(arguments=arguments): + original = self._installed_candidate_with_retained_lkg() + result = self._run("healthy", arguments=arguments) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertNotIn("GENERATING_TOKEN", result.stdout) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + + def test_accept_installed_never_repairs_failed_installed_check(self): + original = self._installed_candidate_with_retained_lkg() + installed = self.state_file.read_bytes() + result = self._run("healthy", policy_state="corrupt", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + self.assertEqual(self.state_file.read_bytes(), installed) + self.assertFalse(self.health_observed_lkg.exists()) + + def test_accept_installed_rejects_image_free_or_staged_configuration(self): + for images in (None, {}, []): + with self.subTest(images=images): + original = self._installed_candidate_with_retained_lkg() + fleet = {} if images is None else {"managed_images": images} + (self.config_repo / "fleet.json").write_text(json.dumps(fleet), encoding="utf-8") + self.assertEqual(git("add", "fleet.json", cwd=self.config_repo).returncode, 0) + self.assertEqual(git("commit", "--allow-empty", "-qm", "unactivated fixture", cwd=self.config_repo).returncode, 0) + self.desired_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + self.env["REMOTE_FIXTURE_REF"] = self.desired_ref + self._write_installed_ref(self.desired_ref) + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("activated managed-image configuration", result.stderr) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + + def test_accept_installed_requires_lkg_persistence_and_readback(self): + for failure in ("write", "readback"): + with self.subTest(failure=failure): + if (self.lkg_dir / "metadata.json").is_dir(): + (self.lkg_dir / "metadata.json").rmdir() + self._installed_candidate_with_retained_lkg() + if failure == "write": + path = self.lkg_dir / "metadata.json" + path.unlink() + path.mkdir() + else: + self._write_executable(self.td / "bin" / "python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == -c && ${{3:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{5:-}} == "$REMOTE_FIXTURE_REF" ]]; then exit 1; fi +exec {shlex.quote(sys.executable)} "$@" +''') + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--check"]) + self.assertEqual(self._reconcile_state()["status"], "failed") + self.assertNotIn("CONVERGED controller=", result.stdout) + + def test_accept_installed_fetches_only_installed_ref_and_rejects_wrong_result(self): + self._installed_candidate_with_retained_lkg() + fetch_log = self.td / "fetches" + wrapper = self.td / "bin" / "git" + wrapper.write_text(wrapper.read_text().replace( + 'if [[ ${1:-} == -C && ${3:-} == fetch ]]; then', + 'if [[ ${1:-} == -C && ${3:-} == fetch ]]; then\n' + + ' printf "%s\\n" "${@: -1}" >>' + shlex.quote(str(fetch_log))), encoding="utf-8") + self.assertEqual(git("commit", "--allow-empty", "-qm", "later default branch", cwd=self.config_repo).returncode, 0) + newer_ref = git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(fetch_log.read_text().splitlines(), [self.desired_ref]) + self.assertEqual(self._lkg_ref(), self.desired_ref) + + original = self._installed_candidate_with_retained_lkg() + self.env["REMOTE_FIXTURE_REF"] = newer_ref + result = self._run("healthy", arguments=("--accept-installed",)) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("installed acceptance fetch identity mismatch", result.stderr) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_bytes(), original) + def _assert_rolled_back(self, result: subprocess.CompletedProcess, health: str) -> None: self.assertEqual(result.returncode, 3, result.stderr) self.assertEqual([call["args"][0] for call in self._installer_calls()], ["--upgrade", "--upgrade"])