From 8769ac69bfac6454f616609eeaf390d4ccdf0c7a Mon Sep 17 00:00:00 2001 From: convee Date: Mon, 7 Sep 2026 00:12:51 +0800 Subject: [PATCH 1/2] fix: preserve containerd runtimes across Lima reprovision --- docs/TROUBLESHOOTING.md | 28 ++++ scripts/local-cluster.yaml | 28 +++- scripts/migrate-local-containerd-provision.py | 100 ++++++++++++ tests/test_containerd_provision.py | 153 ++++++++++++++++++ 4 files changed, 303 insertions(+), 6 deletions(-) create mode 100644 scripts/migrate-local-containerd-provision.py create mode 100644 tests/test_containerd_provision.py diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index b6e1038..9300a84 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -113,6 +113,34 @@ each other even when their displayed addresses match. reusing a VM. Preserve any legacy Workspace data, then destroy and recreate the local profile. Do not bypass the network compatibility check. +## gVisor disappears after a Lima VM restart + +Older local templates unconditionally regenerated `/etc/containerd/config.toml` +on each boot. The `runsc` binaries remained installed, but the runtime handler +was lost. Current templates preserve existing containerd configuration and +restart the daemon only when bootstrap configuration changes. + +Existing Lima instances keep a private copy of their provision scripts; pulling +new repository code is not enough. Check and migrate that copy during an approved +maintenance window (substitute the exact instance name): + +```bash +python3 scripts/migrate-local-containerd-provision.py sandbox-local --check +limactl stop sandbox-local +python3 scripts/migrate-local-containerd-provision.py sandbox-local +limactl start sandbox-local +KUBECONFIG="$(scripts/local-cluster.sh kubeconfig)" \ + bash scripts/install-gvisor-kubeadm.sh sandbox-local +``` + +The migration refuses running VMs and unfamiliar/customized containerd blocks, +changes only the recognized provision block, and verifies the saved template. +It neither stops the VM itself nor replaces ports, resources, mounts or disks. +The final installer repairs an already-lost handler and restarts containerd and +kubelet if needed; it does not migrate the saved Lima template by itself. Verify +a real gVisor workload after recovery and again after a planned restart. A unit +test of repeated provision is not evidence of a real node reboot. + ## `ErrImageNeverPull` or `ImagePullBackOff` **Symptom.** Control Plane, console, volume agent, or Runtime Pods show diff --git a/scripts/local-cluster.yaml b/scripts/local-cluster.yaml index a01a540..109bd15 100644 --- a/scripts/local-cluster.yaml +++ b/scripts/local-cluster.yaml @@ -45,13 +45,29 @@ provision: apt-get -o Acquire::Retries=5 update -qq apt-get -o Acquire::Retries=5 install -y -qq containerd apt-transport-https ca-certificates curl gpg mkdir -p /etc/containerd /etc/apt/keyrings - containerd config default >/etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml - # apt starts containerd before the generated configuration exists. Restart - # it explicitly so kubelet's CRI cgroup-driver discovery sees systemd from - # its first boot rather than caching containerd's built-in cgroupfs default. + # BEGIN sandbox containerd configuration + # Lima reruns system provisioning on restart. Never replace an existing + # configuration: gVisor handlers, registry settings and operator edits + # are persistent node state, not disposable bootstrap output. + containerd_config_changed=0 + if [ ! -s /etc/containerd/config.toml ]; then + containerd config default >/etc/containerd/config.toml + containerd_config_changed=1 + fi + if grep -q 'SystemdCgroup = false' /etc/containerd/config.toml; then + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + containerd_config_changed=1 + fi + # apt starts containerd before first-boot configuration exists. Restart + # only when it changed, so CRI sees systemd without restarting a healthy + # daemon on every repeat provision. systemctl enable containerd - systemctl restart containerd + if [ "$containerd_config_changed" -eq 1 ]; then + systemctl restart containerd + else + systemctl start containerd + fi + # END sandbox containerd configuration curl --retry 5 --retry-all-errors --retry-delay 2 -fsSL https://pkgs.k8s.io/core:/stable:/v1.36/deb/Release.key \ | gpg --dearmor --yes -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.36/deb/ /' \ diff --git a/scripts/migrate-local-containerd-provision.py b/scripts/migrate-local-containerd-provision.py new file mode 100644 index 0000000..4644c92 --- /dev/null +++ b/scripts/migrate-local-containerd-provision.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Update only the known old containerd provision block of a stopped Lima VM. + +Lima stores its own template copy. Editing scripts/local-cluster.yaml alone +does not repair existing VMs. This command never stops or starts a VM and +does not install gVisor or modify the guest filesystem. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess +import textwrap + + +BEGIN = "# BEGIN sandbox containerd configuration" +END = "# END sandbox containerd configuration" +LEGACY = """containerd config default >/etc/containerd/config.toml +sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml +# apt starts containerd before the generated configuration exists. Restart +# it explicitly so kubelet's CRI cgroup-driver discovery sees systemd from +# its first boot rather than caching containerd's built-in cgroupfs default. +systemctl enable containerd +systemctl restart containerd +""" + + +def current_block() -> str: + template = Path(__file__).with_name("local-cluster.yaml").read_text(encoding="utf-8") + lines = template.splitlines(keepends=True) + start = next(i for i, line in enumerate(lines) if line.strip() == BEGIN) + end = next(i for i, line in enumerate(lines[start:], start) if line.strip() == END) + return textwrap.dedent("".join(lines[start:end + 1])) + + +def provision_updates(config: dict) -> list[tuple[int, str]]: + updates = [] + replacement = current_block() + for index, provision in enumerate(config.get("provision", [])): + script = provision.get("script", "") + if not isinstance(script, str): + raise ValueError("provision script is not text; review the VM template manually") + if BEGIN in script: + if replacement not in script: + raise ValueError("managed containerd block was customized; review it manually") + continue + if "containerd config default" not in script: + continue + if script.count(LEGACY) != 1: + raise ValueError("unknown containerd provision block; refusing to replace operator changes") + updates.append((index, script.replace(LEGACY, replacement, 1))) + return updates + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("vm", help="exact Lima VM name") + parser.add_argument("--check", action="store_true", help="report migration needs without editing or stopping anything") + arguments = parser.parse_args() + result = subprocess.run(["limactl", "list", arguments.vm, "--json"], capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError("cannot inspect the requested Lima VM") + record = json.loads(result.stdout) + if record.get("name") != arguments.vm: + raise ValueError("Lima did not return the exact requested VM") + updates = provision_updates(record.get("config", {})) + if arguments.check: + print(json.dumps({"vm": arguments.vm, "migration_required": bool(updates), "blocks": len(updates)})) + return 0 + if not updates: + print(json.dumps({"vm": arguments.vm, "migrated": False, "reason": "no legacy block"})) + return 0 + if record.get("status") != "Stopped": + raise RuntimeError("stop the VM during an approved maintenance window, then rerun migration; no changes made") + command = ["limactl", "edit", arguments.vm, "--tty=false"] + for index, script in updates: + command.extend(["--set", f".provision[{index}].script = {json.dumps(script)}"]) + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError("Lima edit failed; inspect the VM configuration before restarting") + # Readback proves the saved instance template, not merely our repo file. + result = subprocess.run(["limactl", "list", arguments.vm, "--json"], capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError("cannot verify the saved Lima VM configuration") + saved = json.loads(result.stdout) + for index, script in updates: + if saved.get("config", {}).get("provision", [])[index].get("script") != script: + raise RuntimeError("saved Lima provision differs from the requested migration") + print(json.dumps({"vm": arguments.vm, "migrated": True, "blocks": len(updates), "verified": True})) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, RuntimeError, IndexError) as error: + # Do not echo captured VM metadata or the full script (operator scripts + # may contain credentials); our errors contain only action guidance. + raise SystemExit(str(error)) from None diff --git a/tests/test_containerd_provision.py b/tests/test_containerd_provision.py new file mode 100644 index 0000000..d71e9a8 --- /dev/null +++ b/tests/test_containerd_provision.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "containerd_migration", ROOT / "scripts/migrate-local-containerd-provision.py", +) +MIGRATION = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MIGRATION) + + +class ContainerdProvisionTests(unittest.TestCase): + def test_repeated_provision_preserves_gvisor_and_operator_settings(self): + for version, plugin in ((2, "io.containerd.grpc.v1.cri"), (3, "io.containerd.cri.v1.runtime"), (4, "io.containerd.cri.v1.runtime")): + with self.subTest(version=version), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + config = root / "config.toml" + log = root / "commands.log" + tools = root / "bin" + tools.mkdir() + defaults = f'version = {version}\nSystemdCgroup = false\n' + self._tool(tools, "containerd", f"import os\nfrom pathlib import Path\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write('generate\\n')\nprint({defaults!r},end='')\n") + self._tool(tools, "systemctl", "import os,sys\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write('systemctl '+' '.join(sys.argv[1:])+'\\n')\n") + # The guest is Linux; preserve real sed behavior on a macOS + # test host, whose -i spelling requires an empty suffix. + self._tool(tools, "sed", "import os,sys\na=sys.argv[1:]\nif sys.platform=='darwin' and a[0]=='-i': a.insert(1,'')\nos.execv('/usr/bin/sed',['sed',*a])\n") + script = "set -eu\n" + MIGRATION.current_block().replace("/etc/containerd/config.toml", str(config)) + environment = {**os.environ, "PATH": f"{tools}:{os.environ['PATH']}", "PROVISION_LOG": str(log)} + self._provision(script, environment) + self.assertEqual(config.read_text(), defaults.replace("false", "true")) + custom = ( + config.read_text() + f'\n[plugins."{plugin}".containerd.runtimes.runsc]\n' + 'runtime_type = "io.containerd.runsc.v1"\n' + '# Operator-owned registry configuration\n[registry]\nconfig_path = "/etc/containerd/certs.d"\n' + ) + config.write_text(custom) + before = config.stat().st_mtime_ns + self._provision(script, environment) + self._provision(script, environment) + self.assertEqual(config.read_text(), custom) + self.assertEqual(config.stat().st_mtime_ns, before) + commands = log.read_text().splitlines() + self.assertEqual(commands.count("generate"), 1) + self.assertEqual(commands.count("systemctl restart containerd"), 1) + self.assertEqual(commands.count("systemctl start containerd"), 2) + + def test_existing_cgroup_fix_preserves_runtime_and_restarts_once(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + config = root / "config.toml" + log = root / "commands.log" + content = 'version = 2\nSystemdCgroup = false\n[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]\nruntime_type = "io.containerd.runsc.v1"\n' + config.write_text(content) + self._tool(root, "containerd", "raise SystemExit('Existing configuration must not be regenerated')\n") + self._tool(root, "systemctl", "import os,sys\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write(' '.join(sys.argv[1:])+'\\n')\n") + self._tool(root, "sed", "import os,sys\na=sys.argv[1:]\nif sys.platform=='darwin' and a[0]=='-i': a.insert(1,'')\nos.execv('/usr/bin/sed',['sed',*a])\n") + script = "set -eu\n" + MIGRATION.current_block().replace("/etc/containerd/config.toml", str(config)) + environment = {**os.environ, "PATH": f"{root}:{os.environ['PATH']}", "PROVISION_LOG": str(log)} + self._provision(script, environment) + self._provision(script, environment) + self.assertEqual(config.read_text(), content.replace("false", "true")) + self.assertEqual(log.read_text().splitlines().count("restart containerd"), 1) + + @staticmethod + def _tool(directory, name, source): + path = directory / name + path.write_text(f"#!{sys.executable}\n" + source) + path.chmod(0o755) + + def _provision(self, script, environment): + result = subprocess.run(["sh", "-c", script], env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +class ContainerdMigrationTests(unittest.TestCase): + def _record(self, status="Stopped"): + return {"name": "fixture", "status": status, "config": { + "cpus": 7, "memory": "11GiB", "mounts": [{"location": "/operator-owned"}], + "portForwards": [{"guestPort": 6443, "hostPort": 18448}], + "provision": [{"mode": "system", "script": "before\n" + MIGRATION.LEGACY + "after\n"}, + {"mode": "user", "script": "echo operator-script\n"}], + }} + + def test_migration_preserves_unrelated_configuration_and_is_idempotent(self): + record = self._record() + original = copy.deepcopy(record) + updates = MIGRATION.provision_updates(record["config"]) + self.assertEqual(len(updates), 1) + self.assertEqual(record, original, "planning must not mutate the source") + index, script = updates[0] + self.assertEqual(index, 0) + self.assertEqual(script, "before\n" + MIGRATION.current_block() + "after\n") + record["config"]["provision"][index]["script"] = script + self.assertEqual(MIGRATION.provision_updates(record["config"]), []) + + def test_unknown_legacy_changes_are_not_overwritten(self): + for script in ( + MIGRATION.LEGACY.replace("systemctl restart", "custom-systemctl restart"), + MIGRATION.LEGACY.replace("default >", "default > "), + ): + with self.subTest(script=script): + record = self._record() + record["config"]["provision"][0]["script"] = script + with self.assertRaisesRegex(ValueError, "operator changes"): + MIGRATION.provision_updates(record["config"]) + + def test_running_vm_requires_explicit_maintenance_stop(self): + with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run") as run: + run.return_value = subprocess.CompletedProcess([], 0, json.dumps(self._record("Running")), "") + with self.assertRaisesRegex(RuntimeError, "stop the VM"): + MIGRATION.main() + self.assertEqual(run.call_count, 1, "must not edit, stop, or start the VM") + + def test_check_does_not_edit_a_running_vm(self): + with patch.object(sys, "argv", ["migration", "fixture", "--check"]), patch.object(MIGRATION.subprocess, "run") as run: + run.return_value = subprocess.CompletedProcess([], 0, json.dumps(self._record("Running")), "") + self.assertEqual(MIGRATION.main(), 0) + self.assertEqual(run.call_count, 1) + + def test_stopped_vm_edit_is_verified_by_readback(self): + record = self._record() + saved = copy.deepcopy(record) + saved["config"]["provision"][0]["script"] = MIGRATION.provision_updates(record["config"])[0][1] + responses = [subprocess.CompletedProcess([], 0, json.dumps(value), "") for value in (record, {}, saved)] + with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run", side_effect=responses) as run: + self.assertEqual(MIGRATION.main(), 0) + edit = run.call_args_list[1].args[0] + self.assertEqual(edit[:4], ["limactl", "edit", "fixture", "--tty=false"]) + self.assertEqual(edit[4], "--set") + self.assertEqual(json.loads(edit[5].split(" = ", 1)[1]), saved["config"]["provision"][0]["script"]) + self.assertEqual(len(run.call_args_list), 3) + + def test_failed_readback_is_not_reported_as_migrated(self): + record = self._record() + responses = [subprocess.CompletedProcess([], 0, json.dumps(value), "") for value in (record, {}, record)] + with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run", side_effect=responses): + with self.assertRaisesRegex(RuntimeError, "differs"): + MIGRATION.main() + + +if __name__ == "__main__": + unittest.main() From 09c1a7b694a19a73c6ab4c9771293e858917530b Mon Sep 17 00:00:00 2001 From: convee Date: Mon, 7 Sep 2026 00:21:09 +0800 Subject: [PATCH 2/2] docs: synchronize standalone test count after reboot regressions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f26b8e..6191257 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ operation fails; it never falls back to running on the host. ```bash make bootstrap # create .venv and install SDK + test dependencies -make test # 870 unit and contract tests, no network, no cluster +make test # 878 unit and contract tests, no network, no cluster make verify # complete Python, Console, manifest, Helm, wheel gate make help # every Make target with its one-line description ```