diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md index decaa0af..183f5e67 100644 --- a/docs/ADDING-A-HOST.md +++ b/docs/ADDING-A-HOST.md @@ -139,6 +139,10 @@ Use another host or location when it improves failure tolerance. Give every adde ## 10. Drain, replace, or remove a host +For a held drained controller whose recovery commit is not default-branch HEAD, +see [explicit drained recovery](EXPLICIT-DRAINED-RECOVERY.md). It requires a +reviewed supporting manager and cannot admit capacity or change engine/policy. + 1. Merge a private configuration change setting the controller to `drained`. 2. Apply that exact commit with `install-worker-controller.sh --upgrade`. 3. Confirm zero managed runners and run scoped cleanup. diff --git a/docs/EXPLICIT-DRAINED-RECOVERY.md b/docs/EXPLICIT-DRAINED-RECOVERY.md new file mode 100644 index 00000000..076196ca --- /dev/null +++ b/docs/EXPLICIT-DRAINED-RECOVERY.md @@ -0,0 +1,96 @@ +# Explicit drained recovery + +The normal reconcile timer still follows the configuration repository's default +branch. It does not automatically choose historical or recovery commits. +`--installed-ref` remains check-only. Neither option authorizes admission. + +A supporting, independently reviewed manager can instead reconcile one immutable +configuration commit without moving the default branch: + +```bash +sudo /opt/ci-fleet/manager/current/scripts/remote-reconcile.sh \ + --ref 1111111111111111111111111111111111111111 +``` + +Replace the fictional SHA with the full lowercase 40-hex reviewed configuration +commit. This is a **mutating** operation, not a preview. Symbolic, abbreviated, +uppercase, malformed, missing and duplicate refs are rejected before credentials, +fetch or the installer lock. `--check-only`, `--installed-ref` and `--no-op` cannot +be combined with `--ref`. + +## Allowed scope + +- Installed metadata, the installed Git-authored baseline and the candidate must + all identify the same drained controller. Active and disabled candidates are + rejected; metadata alone cannot turn an active authored baseline into drained. +- The candidate must preserve the complete controller and fleet policy, including + configured minimum/maximum capacity (not merely effective drained capacity), + resources, pool budget, routing, network, reporting and organization policy. + This is a generic unchanged-policy rule, not a special one-runner exception. +- Engine repository and full engine ref must match installed state and the current + runtime marker. This mode cannot promote or downgrade the runtime engine. +- The only permitted fleet-document addition is reviewed `managed_images` for + the unchanged current engine. Existing image identities cannot be replaced or + removed, and unrelated engines cannot be added. Installer image publication, + validation and health checks are not bypassed. +- An existing LKG must identify the installed drained baseline. An older active or + otherwise different LKG blocks recovery rather than becoming a rollback target. + Do not edit metadata to evade this rule. Resolve missing or conflicting recovery + evidence through a separately reviewed installation/recovery procedure. + +## Validation, failure and recovery + +The manager fetches the requested commit and checks its exact identity, then +fetches the installed baseline from the same repository. Both remain local to the +transaction. Default-branch state is not rewritten. Existing host approval and +required-check policy remain fail-closed; a commit not accepted by that policy +cannot use this command as a bypass. + +Candidate validation, strict template validation, secret scanning and installer +checks still execute under the existing shared installer lock. Even a request +for the already-installed commit takes the validation/installer/full-health path; +there is no early convergence shortcut. Success requires an `applied` transaction +receipt, installed identity/runtime readback, state-aware full local health and +LKG persistence with readback. Probe delivery is suppressed as in normal reconcile. + +Rejected policy and pre-transaction failures do not apply a fallback or overwrite +an accepted LKG. Unknown or unverified installer outcomes remain failures: do not +infer safe rollback from a nonzero exit alone. Final health rejection restores +only the verified same-engine drained baseline through the existing installer +LKG path. LKG write failure is not suppressed or reported as success; explicit +recovery attempts drained-baseline restoration and returns failure. If candidate +metadata was written before readback failed, the manager atomically restores the +retained verified baseline record **before** the installer's live approval and +checkpoint checks; it never synthesizes a replacement recovery identity. A fresh +health report and matching process exit are required for each probe (0 for +healthy/maintenance, 1 for warning, 2 for unhealthy). Missing output, stale output, +execution failure or inconsistent exit/status cannot verify rollback. A rollback +whose health is not drained maintenance/warning is not reported verified. +Inspect the protected reconcile state and installer diagnostics after every +nonzero exit. Failed recovery must remain held; admission needs separate review +and authorization. This command does not establish active-pilot evidence, +isolation proof or permission to publish otherwise unapproved images. + +## Rollout boundary + +This feature requires a new reviewed supporting engine/manager pin and complete +isolated Linux validation, independent review and authorized staging. It is not +a drop-in flag for older pinned managers, including the old `312` manager. This +feature is only for an already-installed drained controller; adding reviewed image +identities does **not** prove bootstrapability. An image-free baseline requires a +matching protected activation checkpoint for the existing installer recovery path. +Updating public source does not change +an installed manager, runtime, checkpoint or private configuration pin. Preserve +compatible checkpoint recovery and stage manager support before using the flag; +never patch pinned guest source or fabricate LKG/approval records. Engine merge, +template publication, private adoption and live-host acceptance are separate gates. + +## Local regression boundary + +`python3 -I -B scripts/test_explicit_recovery.py` uses temporary repositories and +command doubles for authentication, installer, health and policy. It needs Linux +Bash, Git, Python and flock, but no fleet credentials, Docker or live host. It +proves orchestration and fail-closed routing, not actual image builds, installer +transactions, systemd behavior or host acceptance. Complete exact-source/base +validation still follows [local validation](LOCAL-VALIDATION.md) on a disposable +execution machine; a focused green suite is not complete acceptance evidence. diff --git a/scripts/explicit_recovery.py b/scripts/explicit_recovery.py new file mode 100644 index 00000000..ce13cc0c --- /dev/null +++ b/scripts/explicit_recovery.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Fail-closed policy for explicit, same-engine drained reconciliation. + +Called by the locked manager, never a replacement for installer/review validation. +""" +import json +from pathlib import Path +import re +import sys + + +def verify(candidate_path, baseline_path, state_path, runtime_path, lkg_path): + candidate, baseline, state = (json.loads(Path(p).read_text()) for p in + (candidate_path, baseline_path, state_path)) + controller = state["controller"] + old = baseline["controllers"][controller] + new = candidate["controllers"][controller] + engine = state["engine_ref"] + if not re.fullmatch("[0-9a-f]{40}", engine): + raise ValueError("invalid installed engine") + if state["controller_state"] != "drained" or old["state"] != "drained" or new["state"] != "drained": + raise ValueError("explicit recovery requires drained installed and desired state") + if old != new or old["engine_ref"] != engine: + raise ValueError("explicit recovery cannot change controller policy or engine") + if baseline["organization"]["delivery_engine"] != state["engine_repository"]: + raise ValueError("installed engine repository mismatch") + if Path(runtime_path, ".ci-fleet-engine-ref").read_text().strip() != engine: + raise ValueError("current runtime engine mismatch") + # Only additive image publication for the unchanged engine is recoverable. + # Existing image identities, other engines, and every other policy stay fixed. + old_images = baseline.pop("managed_images", {}) + new_images = candidate.pop("managed_images", {}) + if baseline != candidate: + raise ValueError("explicit recovery cannot change fleet policy") + if any(new_images.get(key) != value for key, value in old_images.items()): + raise ValueError("explicit recovery cannot replace reviewed image identities") + if set(new_images) - set(old_images) - {engine}: + raise ValueError("explicit recovery cannot publish unrelated engine images") + metadata = Path(lkg_path) + if metadata.exists() or metadata.is_symlink(): + lkg = json.loads(metadata.read_text()) + if any(lkg[key] != state[key] for key in ("config_repository", "config_ref", "controller")): + raise ValueError("explicit recovery requires installed drained baseline as LKG") + + +def main(): + try: + verify(*sys.argv[1:]) + except (OSError, ValueError, TypeError, KeyError): + print("ERROR: explicit drained recovery policy rejected", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/remote-reconcile.sh b/scripts/remote-reconcile.sh index 086c6260..64f1ca3a 100755 --- a/scripts/remote-reconcile.sh +++ b/scripts/remote-reconcile.sh @@ -25,18 +25,11 @@ token_script=$script_dir/github-app-token.sh reconcile_state_dir=${CI_FLEET_RECONCILE_STATE_DIR:-/var/lib/ci-fleet/reconcile} reconcile_state_file=$reconcile_state_dir/state.json lkg_dir=${CI_FLEET_LKG_DIR:-/var/lib/ci-fleet/last-known-good} -temp_dir=$(mktemp -d) || exit 2 -transaction_result=$temp_dir/transaction-result.json -: >"$transaction_result" -chmod 0600 "$transaction_result" -exec 7>"$transaction_result" -# shellcheck disable=SC2317 # cleanup_temp is invoked indirectly via trap -cleanup_temp() { rm -rf "$temp_dir"; } -trap cleanup_temp EXIT mode=reconcile # reconcile or check-only no_op=false installed_ref=false +explicit_ref= max_attempts=${CI_FLEET_RECONCILE_MAX_ATTEMPTS:-3} # ponytail: one-hour queue cap; align with any future installer transaction timeout. lock_wait_seconds=3600 @@ -49,6 +42,9 @@ usage: Fetches the desired-state repository at the current default-branch HEAD, validates it, and reconciles the controller if a newer commit is available. + --ref SHA Mutating opt-in drained recovery at one full lowercase 40-hex + commit; unchanged controller/fleet policy and runtime engine only. + Incompatible with --check-only, --installed-ref and --no-op. --check-only Validate and report without reconciling. --installed-ref Check the locked installation's commit (check-only only). --no-op Log what would be done without side effects. @@ -57,6 +53,9 @@ EOF while (($#)); do case "$1" in + --ref) + [[ $# -ge 2 && -z "$explicit_ref" && "$2" =~ ^[0-9a-f]{40}$ ]] || { echo 'ERROR: --ref requires one full lowercase commit SHA' >&2; exit 2; } + explicit_ref=$2; shift ;; --check-only) mode=check-only ;; --installed-ref) installed_ref=true ;; --no-op) no_op=true ;; @@ -65,8 +64,18 @@ while (($#)); do esac shift done +[[ -z "$explicit_ref" || ( "$installed_ref" == false && "$no_op" == false && "$mode" == reconcile ) ]] || { echo 'ERROR: --ref is incompatible with --installed-ref, --check-only and --no-op' >&2; exit 2; } [[ "$installed_ref" == false || "$mode" == check-only ]] || { echo 'ERROR: --installed-ref requires --check-only' >&2; exit 2; } +temp_dir=$(mktemp -d) || exit 2 +transaction_result=$temp_dir/transaction-result.json +: >"$transaction_result" +chmod 0600 "$transaction_result" +exec 7>"$transaction_result" +# shellcheck disable=SC2317 # cleanup_temp is invoked indirectly via trap +cleanup_temp() { rm -rf "$temp_dir"; } +trap cleanup_temp EXIT + note() { printf 'RECONCILE %s\n' "$*"; } die() { printf 'RECONCILE_ERROR: %s\n' "$*" >&2 @@ -243,7 +252,9 @@ apply_lkg() { fi local lkg_ref lkg_repo lkg_controller lkg_status=0 outcome - mapfile -t lkg_vals <<<"$(python3 - "$lkg_dir/metadata.json" <<'PY' 2>/dev/null || true + local rollback_metadata=$lkg_dir/metadata.json + [[ -z "$explicit_ref" ]] || rollback_metadata=$temp_dir/recovery-lkg.json + mapfile -t lkg_vals <<<"$(python3 - "$rollback_metadata" <<'PY' 2>/dev/null || true import json, sys d = json.load(open(sys.argv[1])) for k in ("config_ref", "config_repository", "controller"): @@ -297,8 +308,8 @@ PY save_lkg() { local commit=$1 replace=${2:-false} - install -d -m 0700 "$lkg_dir" - python3 - "$lkg_dir/metadata.json" "$installed_config_repo" "$commit" "$installed_controller" "$replace" <<'PY' 2>/dev/null || true + install -d -m 0700 "$lkg_dir" || return 1 + python3 - "$lkg_dir/metadata.json" "$installed_config_repo" "$commit" "$installed_controller" "$replace" <<'PY' 2>/dev/null import json, os, stat, sys, tempfile path = sys.argv[1] @@ -337,14 +348,28 @@ try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(meta, f, indent=2, sort_keys=True) f.write("\n") + f.flush() + os.fsync(f.fileno()) os.chmod(tmp, 0o600) os.replace(tmp, path) + with open(path, encoding="utf-8") as handle: + if json.load(handle) != meta: + raise OSError("LKG persistence readback mismatch") except: os.unlink(tmp, missing_ok=True) raise PY } +lkg_readback_matches() { + python3 -c ' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +if [value.get(key) for key in ("config_repository", "config_ref", "controller")] != sys.argv[2:]: + raise SystemExit(1) +' "$lkg_dir/metadata.json" "$installed_config_repo" "$1" "$installed_controller" +} + transaction_outcome() { python3 - "$transaction_result" "${1:-outcome}" <<'PY' 2>/dev/null || printf 'unknown\n' import json, sys @@ -395,7 +420,12 @@ PY # --- Health (with rendered env) --- run_health_check() { - local output=$1 + local output=$1 probe_status=0 + # Every invocation must produce its own report, including rollback probes. + if ! rm -f -- "$output"; then + printf 'unknown\n' + return + fi ( set -a # shellcheck disable=SC1090 @@ -403,8 +433,34 @@ run_health_check() { set +a export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=1 python3 "$repo_root/scripts/health.py" local --output "$output" >/dev/null 2>&1 - ) || true - python3 -c "import json; print(json.load(open('$output'))['status'])" 2>/dev/null || echo "unknown" + ) || probe_status=$? + # health.py uses 0 for healthy/maintenance, 1 for warning, 2 for + # unhealthy. A nonzero warning is valid only with its fresh warning report. + python3 - "$output" "$probe_status" <<'PY' 2>/dev/null || printf 'unknown\n' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +status = value["status"] +expected = {"healthy": 0, "maintenance": 0, "warning": 1, "unhealthy": 2}[status] +if int(sys.argv[2]) != expected: + raise SystemExit(1) +print(status) +PY +} + +verify_explicit_rollback_health() { + if [[ -n "$explicit_ref" ]]; then + [[ ${1:-unknown} == maintenance || ${1:-unknown} == warning ]] || return 1 + load_installed_state || return 1 + [[ "$installed_config_ref" == "$restored_config_ref" && "$installed_controller_state" == drained ]] || return 1 + python3 "$script_dir/explicit_recovery.py" "$temp_dir/recovery-baseline.json" "$temp_dir/recovery-baseline.json" \ + "$state_file" "$config_review_root/opt/ci-fleet/current" "$temp_dir/nonexistent-lkg" || return 1 + fi + [[ -z "$explicit_ref" ]] || python3 - "$temp_dir/health.json" <<'PY' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +if value.get("desired_state") != "drained" or value.get("status") not in {"maintenance", "warning"}: + raise SystemExit(1) +PY } # --- Main --- @@ -421,6 +477,10 @@ flock -w "$lock_wait_seconds" 9 || die "timed out waiting for another reconcile load_installed_state || die "no installed state found at $state_file" [[ "$installed_config_ref" =~ ^[0-9a-f]{40}$ ]] || die "installed config ref is not a full lowercase commit SHA" fetch_ref=HEAD +if [[ -n "$explicit_ref" ]]; then + [[ "$installed_controller_state" == drained ]] || die 'explicit recovery requires a drained installation' + fetch_ref=$explicit_ref +fi [[ "$installed_ref" == false ]] || fetch_ref=$installed_config_ref note "INSTALLED controller=${installed_controller} config_repo=${installed_config_repo} config_ref=${installed_config_ref}" @@ -459,6 +519,14 @@ while ((attempt < max_attempts)); do done note "RESOLVED desired_ref=${desired_commit}" +if [[ -n "$explicit_ref" ]]; then + [[ "$desired_commit" == "$explicit_ref" ]] || die 'explicit ref fetch identity mismatch' + git -C "$temp_dir/config-repo" show "$desired_commit:fleet.json" >"$temp_dir/recovery-candidate.json" || die 'candidate unavailable' + fetch_remote_config "$installed_config_repo" "$token" "$installed_config_ref" >/dev/null || die 'drained baseline unavailable' + git -C "$temp_dir/config-repo" show "$installed_config_ref:fleet.json" >"$temp_dir/recovery-baseline.json" || die 'baseline unavailable' + python3 "$script_dir/explicit_recovery.py" "$temp_dir/recovery-candidate.json" "$temp_dir/recovery-baseline.json" \ + "$state_file" "$config_review_root/opt/ci-fleet/current" "$lkg_dir/metadata.json" || die 'explicit recovery policy rejected' +fi if [[ -e "$config_review_policy" || -L "$config_review_policy" || -e "$config_review_required" || -L "$config_review_required" ]]; then desired_tree=$(git -C "$temp_dir/config-repo" rev-parse "$desired_commit^{tree}") || die 'configuration tree is unavailable' @@ -470,7 +538,7 @@ if [[ -e "$config_review_policy" || -L "$config_review_policy" || -e "$config_re fi # Compare with installed -if [[ "$desired_commit" == "$installed_config_ref" ]]; then +if [[ "$desired_commit" == "$installed_config_ref" && -z "$explicit_ref" ]]; then # Same commit — just verify convergence note "NO_CHANGE desired=${desired_commit}" if [[ "$no_op" == true ]]; then @@ -527,7 +595,11 @@ if [[ "$no_op" == true ]]; then fi # Preserve an existing accepted LKG; bootstrap it only when absent or invalid. -save_lkg "$installed_config_ref" +save_lkg "$installed_config_ref" || die 'cannot persist baseline LKG' +if [[ -n "$explicit_ref" ]]; then + lkg_readback_matches "$installed_config_ref" || die 'drained baseline LKG readback rejected' + cp "$lkg_dir/metadata.json" "$temp_dir/recovery-lkg.json" || die 'cannot retain verified recovery baseline' +fi restored_config_ref=$(python3 - "$lkg_dir/metadata.json" <<'PY' import json, sys value = json.load(open(sys.argv[1], encoding="utf-8")) @@ -554,10 +626,16 @@ if CI_FLEET_TRANSACTION_RESULT_FD=7 CI_FLEET_INSTALLER_LOCK_FD=9 "$installer" -- --ref "$desired_commit" \ --controller "$installed_controller" 2>"$temp_dir/upgrade_err"; then note "RECONCILED controller=${installed_controller} config_ref=${desired_commit}" + [[ -z "$explicit_ref" || $(transaction_outcome) == applied ]] || die 'explicit recovery apply outcome unverified' # Run health check save_reconcile_state 'converged' "$desired_commit" "$desired_commit" 'unknown' "reconciled to ${desired_commit}; checking health" load_installed_state || installed_controller_state=unknown + if [[ -n "$explicit_ref" ]]; then + [[ "$installed_config_ref" == "$desired_commit" && "$installed_controller_state" == drained ]] || die 'explicit recovery installed state readback rejected' + python3 "$script_dir/explicit_recovery.py" "$temp_dir/recovery-candidate.json" "$temp_dir/recovery-candidate.json" \ + "$state_file" "$config_review_root/opt/ci-fleet/current" "$temp_dir/nonexistent-lkg" || die 'explicit recovery runtime readback rejected' + fi health_status=$(run_health_check "$temp_dir/health.json") health_desired_state=$(python3 - "$temp_dir/health.json" <<'PY' 2>/dev/null || echo unknown import json, sys @@ -574,13 +652,46 @@ PY save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}; final health rejected" apply_lkg || die "final health was ${health_status}; rollback to last-known-good also failed" rollback_health_status=$(run_health_check "$temp_dir/health.json") + verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified' save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" "final health ${health_status} for ${health_desired_state} rejected for applied state ${installed_controller_state}; rolled back to ${restored_config_ref}" note "ROLLBACK_OK controller=${installed_controller} restored=${restored_config_ref}" exit 3 ;; esac - save_lkg "$desired_commit" true + if ! { save_lkg "$desired_commit" true && lkg_readback_matches "$desired_commit"; }; then + save_reconcile_state 'failed' "$desired_commit" "$desired_commit" "$health_status" 'candidate LKG persistence failed' + if [[ -n "$explicit_ref" ]]; then + # Candidate promotion may already have replaced the live record before a + # readback failure. Restore the retained, verified record atomically before + # the installer consults it for approval AND image-free checkpoint lookup. + python3 - "$temp_dir/recovery-lkg.json" "$lkg_dir/metadata.json" <<'PY' || die 'baseline metadata restoration failed' +import os, sys, tempfile +source, target = sys.argv[1:] +original = open(source, "rb").read() +fd, temporary = tempfile.mkstemp(prefix=".lkg-restore.", dir=os.path.dirname(target)) +try: + with os.fdopen(fd, "wb") as handle: + handle.write(original) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, target) + if open(target, "rb").read() != original: + raise OSError("baseline restoration readback mismatch") +finally: + if os.path.exists(temporary): + os.unlink(temporary) +PY + lkg_readback_matches "$restored_config_ref" || die 'baseline metadata restoration readback rejected' + apply_lkg || die 'LKG persistence failed; drained baseline rollback failed' + rollback_health_status=$(run_health_check "$temp_dir/health.json") + verify_explicit_rollback_health "$rollback_health_status" || die 'drained rollback health unverified' + [[ "$rollback_health_status" == maintenance || "$rollback_health_status" == warning ]] || die 'LKG persistence failed; rollback health unverified' + save_reconcile_state 'rolled_back' "$desired_commit" "$restored_config_ref" "$rollback_health_status" 'LKG persistence failed; restored drained baseline' + fi + exit 3 + fi save_reconcile_state 'converged' "$desired_commit" "$desired_commit" "$health_status" "reconciled to ${desired_commit}" true note "RECONCILE_OK controller=${installed_controller} desired=${desired_commit} applied=${desired_commit} health=${health_status}" exit 0 @@ -602,6 +713,7 @@ else fi if [[ "$upgrade_status" == 20 && "$outcome" == rollback_verified ]]; then health_status=$(run_health_check "$temp_dir/health.json") + verify_explicit_rollback_health "$health_status" || die 'installer drained rollback readback unverified' save_reconcile_state 'rolled_back' "$desired_commit" "$installed_config_ref" "$health_status" "reconcile failed; installer verified rollback to ${installed_config_ref}" note "ROLLBACK_OK controller=${installed_controller} restored=${installed_config_ref}" exit 3 diff --git a/scripts/test_explicit_recovery.py b/scripts/test_explicit_recovery.py new file mode 100644 index 00000000..c1532a5f --- /dev/null +++ b/scripts/test_explicit_recovery.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +"""Isolated exact-ref recovery command fixtures; no host or network access.""" +import importlib.util +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("reconcile_tests", Path(__file__).with_name("test_remote_reconcile.py")) +assert spec is not None and spec.loader is not None +fixtures = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixtures) + + +class ExplicitRecovery(fixtures.TestRemoteReconcileTransaction): + # Reuse fixture construction, not the parent regression cases. + def setUp(self): + # The real approval reader rejects world-writable ancestors (e.g. /tmp). + # Keep this suite's isolated roots under a private home directory. + private = tempfile.TemporaryDirectory(prefix="ci-fleet-recovery-", dir=Path.home()) + self.addCleanup(private.cleanup) + with patch.object(tempfile, "tempdir", private.name): + super().setUp() + self.engine = "a" * 40 + self.config = {"organization": {"delivery_engine": "RandomDevelopment/ci-fleet"}, + "runner_pools": {"ordinary": {"capacity_budget": 1}}, + "controllers": {"example-ci-01": {"state": "drained", "engine_ref": self.engine, + "pool": "ordinary", "min_runners": 0, "max_runners": 1, + "runner_resources": {"cpu_cores": 1, "memory_mib": 1024}}}} + self.baseline = self.commit(self.config) + self.candidate = self.commit(self.config) + active = json.loads(json.dumps(self.config)) + active["controllers"]["example-ci-01"]["state"] = "active" + self.default = self.commit(active) + state = {"config_repository": "example-org/private-config", "config_ref": self.baseline, + "controller": "example-ci-01", "controller_state": "drained", "engine_ref": self.engine, + "engine_repository": "RandomDevelopment/ci-fleet"} + self.state_file.write_text(json.dumps(state)) + installer = self.root / "scripts/install-worker-controller.sh" + installer.write_text(installer.read_text().replace("exit 0\n", '''if [[ ${1:-} == --upgrade && ${FAKE_SKIP_STATE_UPDATE:-0} != 1 ]]; then + python3 - "$CI_FLEET_REMOTE_STATE_FILE" "$@" <<'PY' +import json, sys +path = sys.argv[1] +value = json.load(open(path)) +value["config_ref"] = sys.argv[sys.argv.index("--ref") + 1] +with open(path, "w") as handle: + json.dump(value, handle) +PY +fi +exit 0 +''')) + self.env["CI_FLEET_ROOT_PREFIX"] = str(self.td) + runtime = self.td / "opt/ci-fleet/current" + runtime.mkdir(parents=True) + (runtime / ".ci-fleet-engine-ref").write_text(self.engine + "\n") + helper = Path(__file__).with_name("explicit_recovery.py") + if helper.exists(): + shutil.copy2(helper, self.root / "scripts/explicit_recovery.py") + real_git = shutil.which("git") + assert real_git is not None + self._write_executable(self.td / "bin/git", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == -C && ${{3:-}} == fetch ]]; then + printf '%s\\n' "${{@: -1}}" >> "$FETCH_LOG" + exec {shlex.quote(real_git)} -C "$2" fetch -q "$REMOTE_FIXTURE_REPO" "${{@: -1}}" +fi +exec {shlex.quote(real_git)} "$@" +''') + self.env.update(FETCH_LOG=str(self.td / "fetch-log"), FAKE_HEALTH_STATUS="maintenance", + FAKE_HEALTH_DESIRED_STATE="drained", FAKE_ROLLBACK_HEALTH_STATUS="maintenance", + FAKE_ROLLBACK_HEALTH_DESIRED_STATE="drained") + # Preserve a real, accepted baseline metadata record byte-for-byte on failure. + self.old_lkg = json.dumps({"config_repository": state["config_repository"], "config_ref": self.baseline, + "controller": state["controller"], "saved_at": 1}) + (self.lkg_dir / "metadata.json").write_text(self.old_lkg) + (self.lkg_dir / "metadata.json").chmod(0o600) + + def commit(self, config): + (self.config_repo / "fleet.json").write_text(json.dumps(config)) + fixtures.git("add", "fleet.json", cwd=self.config_repo) + result = fixtures.git("commit", "--allow-empty", "-qm", "fixture", cwd=self.config_repo) + self.assertEqual(result.returncode, 0, result.stderr) + return fixtures.git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip() + + def run_ref(self, *args): + return subprocess.run([str(self.root / "scripts/remote-reconcile.sh"), *args], + env=self.env, text=True, capture_output=True, timeout=30) + + def test_exact_ref_not_default_head(self): + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self._lkg_ref(), self.candidate) + self.assertEqual(fixtures.git("rev-parse", "HEAD", cwd=self.config_repo).stdout.strip(), self.default) + calls = self._installer_calls() + self.assertIn(self.candidate, calls[-1]["args"]) + self.assertNotIn("HEAD", (self.td / "fetch-log").read_text().splitlines()) + + + def test_same_ref_requires_health_and_durable_lkg(self): + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + result = self.run_ref("--ref", self.baseline) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("CONVERGED controller", result.stdout) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_success_without_applied_transaction_is_rejected(self): + installer = self.root / "scripts/install-worker-controller.sh" + installer.write_text("\n".join(":" if '"outcome":"applied"' in line else line + for line in installer.read_text().splitlines()) + "\n") + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_rollback_active_report_is_not_verified(self): + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + self.env["FAKE_ROLLBACK_HEALTH_DESIRED_STATE"] = "active" + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + self.assertNotIn("ROLLBACK_OK", result.stdout) + + def test_malformed_and_incompatible_refs_before_effects(self): + cases = [("--ref", value) for value in ("HEAD", "a" * 39, "A" * 40, "g" * 40, "")] + cases += [("--ref", self.candidate, flag) for flag in ("--installed-ref", "--no-op", "--check-only")] + cases += [("--ref",), ("--ref", self.candidate, "--ref", self.candidate)] + self.lock_file.unlink(missing_ok=True) + for args in cases: + with self.subTest(args=args): + result = self.run_ref(*args) + self.assertEqual(result.returncode, 2) + self.assertNotIn("GENERATING_TOKEN", result.stdout) + self.assertFalse(self.lock_file.exists()) + self.assertFalse(self.installer_log.exists()) + self.assertFalse((self.td / "fetch-log").exists()) + + def test_policy_changes_rejected_without_installer_or_lkg_mutation(self): + mutations = [ + ("state", "active"), ("state", "disabled"), ("engine_ref", "b" * 40), + ("max_runners", 2), ("min_runners", 1), ("pool", "other"), + ("runner_resources", {"cpu_cores": 2, "memory_mib": 1024}), + ("scale_set_name", "changed"), ("docker_network_policy", {}), + ] + for key, value in mutations: + with self.subTest(key=key, value=value): + config = json.loads(json.dumps(self.config)) + config["controllers"]["example-ci-01"][key] = value + candidate = self.commit(config) + result = self.run_ref("--ref", candidate) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_active_installed_baseline_rejected_before_token(self): + state = json.loads(self.state_file.read_text()) + state["controller_state"] = "active" + self.state_file.write_text(json.dumps(state)) + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("GENERATING_TOKEN", result.stdout) + + def test_active_authored_baseline_cannot_be_forged_drained_by_metadata(self): + state = json.loads(self.state_file.read_text()) + state["config_ref"] = self.default + self.state_file.write_text(json.dumps(state)) + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.installer_log.exists()) + + def test_current_runtime_engine_mismatch_rejected(self): + (self.td / "opt/ci-fleet/current/.ci-fleet-engine-ref").write_text("b" * 40) + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.installer_log.exists()) + + def test_review_denial_preserves_lkg(self): + self._activate_approval_fixture(reject=True) + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3) + self.assertFalse(self.installer_log.exists()) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_recovery_adds_images_only_for_current_engine(self): + config = json.loads(json.dumps(self.config)) + config["managed_images"] = {self.engine: {"amd64": {"runner": "sha256:" + "1" * 64, + "controller": "sha256:" + "2" * 64}}} + candidate = self.commit(config) + result = self.run_ref("--ref", candidate) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self._lkg_ref(), candidate) + + def test_unknown_transaction_keeps_old_lkg(self): + self.env.update(FAKE_INSTALLER_RESULT="unverified") + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3) + self.assertEqual(len(self._installer_calls()), 1) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_pretransaction_denial_keeps_old_lkg(self): + self.env.update(FAKE_TRANSACTION_RECORD=json.dumps({"schema_version": 1, "outcome": "rejected", + "phase": "pre_transaction", "reason": "installer_precondition"}), FAKE_TRANSACTION_EXIT="2") + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3) + self.assertEqual(self._reconcile_state()["transaction_phase"], "pre_transaction") + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_final_health_failure_rolls_back_to_drained_baseline(self): + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual(self._reconcile_state()["status"], "rolled_back") + self.assertIn(self.baseline, self._installer_calls()[-1]["args"]) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_installer_cannot_silently_keep_old_ref(self): + self.env["FAKE_SKIP_STATE_UPDATE"] = "1" + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_unrelated_fleet_and_image_changes_are_rejected(self): + for mutation in ("routing", "budget", "repository", "unrelated_images", "replace_images"): + with self.subTest(mutation=mutation): + config = json.loads(json.dumps(self.config)) + if mutation == "routing": + config["runner_pools"]["ordinary"]["routing_labels"] = ["changed"] + elif mutation == "budget": + config["runner_pools"]["ordinary"]["capacity_budget"] = 2 + elif mutation == "repository": + config["organization"]["delivery_engine"] = "example-org/other-engine" + elif mutation == "unrelated_images": + config["managed_images"] = {"b" * 40: {}} + else: + baseline_config = json.loads(json.dumps(config)) + baseline_config["managed_images"] = {self.engine: {"amd64": {"runner": "old"}}} + baseline = self.commit(baseline_config) + state = json.loads(self.state_file.read_text()) + state["config_ref"] = baseline + self.state_file.write_text(json.dumps(state)) + config["managed_images"] = {self.engine: {"amd64": {"runner": "new"}}} + result = self.run_ref("--ref", self.commit(config)) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.installer_log.exists()) + + def test_lkg_readback_mismatch_must_not_be_success(self): + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then + # Simulate a write helper returning zero without persisting the candidate. + exit 0 +fi +exec {shlex.quote(real_python)} "$@" +''') + self.env["DENY_LKG_REF"] = self.candidate + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + def test_malformed_ref_creates_no_transaction_temp(self): + marker = self.td / "mktemp-called" + self._write_executable(self.td / "bin/mktemp", f'#!/usr/bin/env bash\ntouch {shlex.quote(str(marker))}\nexit 1\n') + result = self.run_ref("--ref", "HEAD") + self.assertEqual(result.returncode, 2) + self.assertFalse(marker.exists()) + + def test_rollback_cannot_claim_success_without_baseline_readback(self): + installer = self.root / "scripts/install-worker-controller.sh" + installer.write_text(installer.read_text().replace('${FAKE_SKIP_STATE_UPDATE:-0} != 1', + '${FAKE_SKIP_STATE_UPDATE:-0} != 1 && ${2:-} != --restore-last-known-good')) + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + self.assertNotIn("ROLLBACK_OK", result.stdout) + + def install_live_recovery_guards(self): + """Exercise the installer's actual approval and image-free checkpoint guards.""" + source = Path(__file__).with_name("install-worker-controller.sh").read_text() + guards = source.split("find_bootstrap_lkg_checkpoint() {", 1)[1].split("\nvalidate_candidate_config_commit()", 1)[0] + approval = source.split("verify_configuration_approval() {", 1)[1].split("\nrequire_configuration_approval_capability()", 1)[0] + scripts = self.root / "scripts" + for name in ("verify_config_approval.py", "desired_state.py", "apt_cache.py"): + shutil.copy2(Path(__file__).with_name(name), scripts / name) + # Candidate network evidence remains doubled; recovery uses the real CLI. + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == */verify_config_approval.py && " $* " != *" --recovery-record "* ]]; then + exit 0 +fi +if [[ ${{1:-}} == */desired_state.py && ${{2:-}} == validate ]]; then exit 0; fi +exec {shlex.quote(real_python)} "$@" +''') + policy = self.td / "etc/ci-fleet" + policy.mkdir(parents=True) + (policy / "config-review-required").write_text("1\n") + (policy / "config-review-policy.json").write_text(json.dumps({ + "schema_version": 1, "configuration_repository_id": 1, "branch": "main", + "workflow_id": 1, "required_jobs": ["validate"], "reviewer_ids": [1]})) + for path in policy.iterdir(): + path.chmod(0o600) + checkpoints = self.td / "checkpoints" + checkpoint = checkpoints / "baseline" + checkpoint.mkdir(parents=True, mode=0o700) + checkpoints.chmod(0o700) + (checkpoint / ".complete").write_text("1\n") + (checkpoint / "install-state.json").write_text(self.state_file.read_text()) + (checkpoint / "ci-fleet.env").write_text( + f"CI_FLEET_CONFIG_REPOSITORY=example-org/private-config\nCI_FLEET_CONFIG_REF={self.baseline}\nCI_FLEET_INSTANCE=example-ci-01\n") + for path in checkpoint.iterdir(): + path.chmod(0o600) + installer = scripts / "install-worker-controller.sh" + preamble = '''set -e +repo_root=$(cd "$(dirname "$0")/.." && pwd) +config_review_policy=$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/config-review-policy.json +config_review_required=$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/config-review-required +state_file=$CI_FLEET_REMOTE_STATE_FILE +state_root=$(dirname "$state_file") +checkpoints_dir=$CI_FLEET_ROOT_PREFIX/checkpoints +testing=1 +die() { echo "$*" >&2; exit 2; } +configuration_approval_required() { return 0; } +''' + guard_call = '''if [[ ${2:-} == --restore-last-known-good ]]; then + restore_lkg=true + args=("$@") + for ((i=0; i<$#; i++)); do + case ${args[i]} in + --ref) config_ref=${args[i+1]} ;; + --config-identity) config_identity=${args[i+1]} ;; + --controller) controller_id=${args[i+1]} ;; + esac + done + config_tree=$(git -C "$REMOTE_FIXTURE_REPO" rev-parse "$config_ref^{tree}") + verify_configuration_approval + find_bootstrap_lkg_checkpoint > "$CI_FLEET_ROOT_PREFIX/checkpoint-used" +fi +''' + installer.write_text(installer.read_text().replace("python3 - ", preamble + + "find_bootstrap_lkg_checkpoint() {" + guards + "\nverify_configuration_approval() {" + approval + + "\n" + guard_call + "python3 - ", 1)) + + def test_candidate_written_then_readback_failure_restores_live_trusted_baseline(self): + self.install_live_recovery_guards() + wrapper = self.td / "bin/python3" + wrapper.write_text(wrapper.read_text().replace("exec ", '''if [[ ${1:-} == -c && ${3:-} == "$CI_FLEET_LKG_DIR/metadata.json" && ${5:-} == "$DENY_LKG_REF" ]]; then + exit 1 +fi +exec ''', 1)) + self.env["DENY_LKG_REF"] = self.candidate + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual(self._reconcile_state()["status"], "rolled_back") + self.assertEqual(json.loads(self.state_file.read_text())["config_ref"], self.baseline) + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + self.assertTrue((self.td / "checkpoint-used").read_text().strip()) + + def test_live_recovery_guard_still_rejects_missing_checkpoint(self): + self.install_live_recovery_guards() + shutil.rmtree(self.td / "checkpoints") + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + + def test_live_recovery_guard_still_rejects_untrusted_policy(self): + self.install_live_recovery_guards() + (self.td / "etc/ci-fleet/config-review-policy.json").chmod(0o666) + self.env["FAKE_HEALTH_STATUS"] = "unhealthy" + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + + def test_rollback_probe_failure_without_output_cannot_reuse_candidate_report(self): + health = self.root / "scripts/health.py" + health.write_text(health.read_text().replace('with open(output, "w",', + 'if rollback:\n raise SystemExit(1)\nwith open(output, "w",')) + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then exit 1; fi +exec {shlex.quote(real_python)} "$@" +''') + self.env["DENY_LKG_REF"] = self.candidate + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back", result.stdout + result.stderr) + + def test_fresh_accepted_report_with_failed_process_is_rejected(self): + health = self.root / "scripts/health.py" + health.write_text(health.read_text().replace('raise SystemExit({"healthy": 0, "maintenance": 0, "warning": 1, "unhealthy": 2}.get(status, 2))', 'raise SystemExit(1)')) + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotEqual(self._reconcile_state()["status"], "rolled_back") + + def test_warning_probe_exit_one_remains_accepted(self): + self.env["FAKE_HEALTH_STATUS"] = "warning" + result = self.run_ref("--ref", self.candidate) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_lkg_write_failure_never_reports_success(self): + real_python = shutil.which("python3") + assert real_python is not None + self._write_executable(self.td / "bin/python3", f'''#!/usr/bin/env bash +if [[ ${{1:-}} == - && ${{2:-}} == "$CI_FLEET_LKG_DIR/metadata.json" && ${{4:-}} == "$DENY_LKG_REF" ]]; then + exit 1 +fi +exec {shlex.quote(real_python)} "$@" +''') + self.env["DENY_LKG_REF"] = self.candidate + result = self.run_ref("--ref", self.candidate) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("RECONCILE_OK", result.stdout) + self.assertEqual(self._reconcile_state()["status"], "rolled_back") + self.assertEqual((self.lkg_dir / "metadata.json").read_text(), self.old_lkg) + + +# Inherited fixture regression tests belong to their original suite. +for name in list(vars(fixtures.TestRemoteReconcileTransaction)): + if name.startswith("test_"): + setattr(ExplicitRecovery, name, None) + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py index 2bc61145..ddce8770 100644 --- a/scripts/test_remote_reconcile.py +++ b/scripts/test_remote_reconcile.py @@ -453,6 +453,7 @@ def setUp(self): handle.write("{") else: json.dump({"status": status, "desired_state": desired_state}, handle) +raise SystemExit({"healthy": 0, "maintenance": 0, "warning": 1, "unhealthy": 2}.get(status, 2)) """, encoding="utf-8", ) @@ -863,7 +864,7 @@ def test_remote_installer_calls_keep_identity_and_lock(self): def test_same_commit_drift_is_reconciled(self): """Any failed same-commit check falls through to repair.""" content = RECONCILE_SCRIPT.read_text() - same_commit = content.split('if [[ "$desired_commit" == "$installed_config_ref" ]]', 1)[1] + same_commit = content.split('# Compare with installed', 1)[1] same_commit = same_commit.split("# New commit or drift", 1)[0] self.assertNotIn("controller_running", same_commit) self.assertNotIn("internal drift tracked by drift timer", same_commit) diff --git a/scripts/validate.sh b/scripts/validate.sh index d19b23df..a8b6a268 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -41,6 +41,8 @@ python3 -m py_compile \ scripts/test_cleanup_networks.py \ scripts/test_apply_docker_network_policy.py \ scripts/test_remote_reconcile.py \ + scripts/explicit_recovery.py \ + scripts/test_explicit_recovery.py \ scripts/test_health.py \ scripts/test_status_receiver.py \ scripts/test_quickstart.py \ @@ -56,6 +58,7 @@ python3 scripts/test_cleanup_networks.py python3 scripts/test_cleanup_storage.py python3 scripts/test_apply_docker_network_policy.py python3 scripts/test_remote_reconcile.py +python3 -I -B scripts/test_explicit_recovery.py python3 scripts/test_health.py python3 scripts/test_endpoint_snapshot.py python3 scripts/test_endpoint_snapshot_sources.py