fix: preserve gVisor configuration across Lima restarts

README.md

@@ -50,7 +50,7 @@ operation fails; it never falls back to running on the host.
 
 ```bash
 make bootstrap                 # create .venv and install SDK + test dependencies
-make test                      # 870 unit and contract tests, no network, no cluster
+make test                      # 878 unit and contract tests, no network, no cluster
 make verify                    # complete Python, Console, manifest, Helm, wheel gate
 make help                      # every Make target with its one-line description
 ```

docs/TROUBLESHOOTING.md

@@ -113,6 +113,34 @@ each other even when their displayed addresses match.
 reusing a VM. Preserve any legacy Workspace data, then destroy and recreate the
 local profile. Do not bypass the network compatibility check.
 
+## gVisor disappears after a Lima VM restart
+
+Older local templates unconditionally regenerated `/etc/containerd/config.toml`
+on each boot. The `runsc` binaries remained installed, but the runtime handler
+was lost. Current templates preserve existing containerd configuration and
+restart the daemon only when bootstrap configuration changes.
+
+Existing Lima instances keep a private copy of their provision scripts; pulling
+new repository code is not enough. Check and migrate that copy during an approved
+maintenance window (substitute the exact instance name):
+
+```bash
+python3 scripts/migrate-local-containerd-provision.py sandbox-local --check
+limactl stop sandbox-local
+python3 scripts/migrate-local-containerd-provision.py sandbox-local
+limactl start sandbox-local
+KUBECONFIG="$(scripts/local-cluster.sh kubeconfig)" \
+  bash scripts/install-gvisor-kubeadm.sh sandbox-local
+```
+
+The migration refuses running VMs and unfamiliar/customized containerd blocks,
+changes only the recognized provision block, and verifies the saved template.
+It neither stops the VM itself nor replaces ports, resources, mounts or disks.
+The final installer repairs an already-lost handler and restarts containerd and
+kubelet if needed; it does not migrate the saved Lima template by itself. Verify
+a real gVisor workload after recovery and again after a planned restart. A unit
+test of repeated provision is not evidence of a real node reboot.
+
 ## `ErrImageNeverPull` or `ImagePullBackOff`
 
 **Symptom.** Control Plane, console, volume agent, or Runtime Pods show

scripts/local-cluster.yaml

@@ -45,13 +45,29 @@ provision:
       apt-get -o Acquire::Retries=5 update -qq
       apt-get -o Acquire::Retries=5 install -y -qq containerd apt-transport-https ca-certificates curl gpg
       mkdir -p /etc/containerd /etc/apt/keyrings
-      containerd config default >/etc/containerd/config.toml
-      sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
-      # apt starts containerd before the generated configuration exists. Restart
-      # it explicitly so kubelet's CRI cgroup-driver discovery sees systemd from
-      # its first boot rather than caching containerd's built-in cgroupfs default.
+      # BEGIN sandbox containerd configuration
+      # Lima reruns system provisioning on restart. Never replace an existing
+      # configuration: gVisor handlers, registry settings and operator edits
+      # are persistent node state, not disposable bootstrap output.
+      containerd_config_changed=0
+      if [ ! -s /etc/containerd/config.toml ]; then
+        containerd config default >/etc/containerd/config.toml
+        containerd_config_changed=1
+      fi
+      if grep -q 'SystemdCgroup = false' /etc/containerd/config.toml; then
+        sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+        containerd_config_changed=1
+      fi
+      # apt starts containerd before first-boot configuration exists. Restart
+      # only when it changed, so CRI sees systemd without restarting a healthy
+      # daemon on every repeat provision.
       systemctl enable containerd
-      systemctl restart containerd
+      if [ "$containerd_config_changed" -eq 1 ]; then
+        systemctl restart containerd
+      else
+        systemctl start containerd
+      fi
+      # END sandbox containerd configuration
       curl --retry 5 --retry-all-errors --retry-delay 2 -fsSL https://pkgs.k8s.io/core:/stable:/v1.36/deb/Release.key \
         | gpg --dearmor --yes -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
       echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.36/deb/ /' \

scripts/migrate-local-containerd-provision.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Update only the known old containerd provision block of a stopped Lima VM.
+
+Lima stores its own template copy. Editing scripts/local-cluster.yaml alone
+does not repair existing VMs. This command never stops or starts a VM and
+does not install gVisor or modify the guest filesystem.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import subprocess
+import textwrap
+
+
+BEGIN = "# BEGIN sandbox containerd configuration"
+END = "# END sandbox containerd configuration"
+LEGACY = """containerd config default >/etc/containerd/config.toml
+sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+# apt starts containerd before the generated configuration exists. Restart
+# it explicitly so kubelet's CRI cgroup-driver discovery sees systemd from
+# its first boot rather than caching containerd's built-in cgroupfs default.
+systemctl enable containerd
+systemctl restart containerd
+"""
+
+
+def current_block() -> str:
+    template = Path(__file__).with_name("local-cluster.yaml").read_text(encoding="utf-8")
+    lines = template.splitlines(keepends=True)
+    start = next(i for i, line in enumerate(lines) if line.strip() == BEGIN)
+    end = next(i for i, line in enumerate(lines[start:], start) if line.strip() == END)
+    return textwrap.dedent("".join(lines[start:end + 1]))
+
+
+def provision_updates(config: dict) -> list[tuple[int, str]]:
+    updates = []
+    replacement = current_block()
+    for index, provision in enumerate(config.get("provision", [])):
+        script = provision.get("script", "")
+        if not isinstance(script, str):
+            raise ValueError("provision script is not text; review the VM template manually")
+        if BEGIN in script:
+            if replacement not in script:
+                raise ValueError("managed containerd block was customized; review it manually")
+            continue
+        if "containerd config default" not in script:
+            continue
+        if script.count(LEGACY) != 1:
+            raise ValueError("unknown containerd provision block; refusing to replace operator changes")
+        updates.append((index, script.replace(LEGACY, replacement, 1)))
+    return updates
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("vm", help="exact Lima VM name")
+    parser.add_argument("--check", action="store_true", help="report migration needs without editing or stopping anything")
+    arguments = parser.parse_args()
+    result = subprocess.run(["limactl", "list", arguments.vm, "--json"], capture_output=True, text=True, check=False)
+    if result.returncode:
+        raise RuntimeError("cannot inspect the requested Lima VM")
+    record = json.loads(result.stdout)
+    if record.get("name") != arguments.vm:
+        raise ValueError("Lima did not return the exact requested VM")
+    updates = provision_updates(record.get("config", {}))
+    if arguments.check:
+        print(json.dumps({"vm": arguments.vm, "migration_required": bool(updates), "blocks": len(updates)}))
+        return 0
+    if not updates:
+        print(json.dumps({"vm": arguments.vm, "migrated": False, "reason": "no legacy block"}))
+        return 0
+    if record.get("status") != "Stopped":
+        raise RuntimeError("stop the VM during an approved maintenance window, then rerun migration; no changes made")
+    command = ["limactl", "edit", arguments.vm, "--tty=false"]
+    for index, script in updates:
+        command.extend(["--set", f".provision[{index}].script = {json.dumps(script)}"])
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode:
+        raise RuntimeError("Lima edit failed; inspect the VM configuration before restarting")
+    # Readback proves the saved instance template, not merely our repo file.
+    result = subprocess.run(["limactl", "list", arguments.vm, "--json"], capture_output=True, text=True, check=False)
+    if result.returncode:
+        raise RuntimeError("cannot verify the saved Lima VM configuration")
+    saved = json.loads(result.stdout)
+    for index, script in updates:
+        if saved.get("config", {}).get("provision", [])[index].get("script") != script:
+            raise RuntimeError("saved Lima provision differs from the requested migration")
+    print(json.dumps({"vm": arguments.vm, "migrated": True, "blocks": len(updates), "verified": True}))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OSError, ValueError, RuntimeError, IndexError) as error:
+        # Do not echo captured VM metadata or the full script (operator scripts
+        # may contain credentials); our errors contain only action guidance.
+        raise SystemExit(str(error)) from None

tests/test_containerd_provision.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import copy
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location(
+    "containerd_migration", ROOT / "scripts/migrate-local-containerd-provision.py",
+)
+MIGRATION = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MIGRATION)
+
+
+class ContainerdProvisionTests(unittest.TestCase):
+    def test_repeated_provision_preserves_gvisor_and_operator_settings(self):
+        for version, plugin in ((2, "io.containerd.grpc.v1.cri"), (3, "io.containerd.cri.v1.runtime"), (4, "io.containerd.cri.v1.runtime")):
+            with self.subTest(version=version), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                config = root / "config.toml"
+                log = root / "commands.log"
+                tools = root / "bin"
+                tools.mkdir()
+                defaults = f'version = {version}\nSystemdCgroup = false\n'
+                self._tool(tools, "containerd", f"import os\nfrom pathlib import Path\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write('generate\\n')\nprint({defaults!r},end='')\n")
+                self._tool(tools, "systemctl", "import os,sys\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write('systemctl '+' '.join(sys.argv[1:])+'\\n')\n")
+                # The guest is Linux; preserve real sed behavior on a macOS
+                # test host, whose -i spelling requires an empty suffix.
+                self._tool(tools, "sed", "import os,sys\na=sys.argv[1:]\nif sys.platform=='darwin' and a[0]=='-i': a.insert(1,'')\nos.execv('/usr/bin/sed',['sed',*a])\n")
+                script = "set -eu\n" + MIGRATION.current_block().replace("/etc/containerd/config.toml", str(config))
+                environment = {**os.environ, "PATH": f"{tools}:{os.environ['PATH']}", "PROVISION_LOG": str(log)}
+                self._provision(script, environment)
+                self.assertEqual(config.read_text(), defaults.replace("false", "true"))
+                custom = (
+                    config.read_text() + f'\n[plugins."{plugin}".containerd.runtimes.runsc]\n'
+                    'runtime_type = "io.containerd.runsc.v1"\n'
+                    '# Operator-owned registry configuration\n[registry]\nconfig_path = "/etc/containerd/certs.d"\n'
+                )
+                config.write_text(custom)
+                before = config.stat().st_mtime_ns
+                self._provision(script, environment)
+                self._provision(script, environment)
+                self.assertEqual(config.read_text(), custom)
+                self.assertEqual(config.stat().st_mtime_ns, before)
+                commands = log.read_text().splitlines()
+                self.assertEqual(commands.count("generate"), 1)
+                self.assertEqual(commands.count("systemctl restart containerd"), 1)
+                self.assertEqual(commands.count("systemctl start containerd"), 2)
+
+    def test_existing_cgroup_fix_preserves_runtime_and_restarts_once(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            config = root / "config.toml"
+            log = root / "commands.log"
+            content = 'version = 2\nSystemdCgroup = false\n[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]\nruntime_type = "io.containerd.runsc.v1"\n'
+            config.write_text(content)
+            self._tool(root, "containerd", "raise SystemExit('Existing configuration must not be regenerated')\n")
+            self._tool(root, "systemctl", "import os,sys\nwith open(os.environ['PROVISION_LOG'],'a') as f: f.write(' '.join(sys.argv[1:])+'\\n')\n")
+            self._tool(root, "sed", "import os,sys\na=sys.argv[1:]\nif sys.platform=='darwin' and a[0]=='-i': a.insert(1,'')\nos.execv('/usr/bin/sed',['sed',*a])\n")
+            script = "set -eu\n" + MIGRATION.current_block().replace("/etc/containerd/config.toml", str(config))
+            environment = {**os.environ, "PATH": f"{root}:{os.environ['PATH']}", "PROVISION_LOG": str(log)}
+            self._provision(script, environment)
+            self._provision(script, environment)
+            self.assertEqual(config.read_text(), content.replace("false", "true"))
+            self.assertEqual(log.read_text().splitlines().count("restart containerd"), 1)
+
+    @staticmethod
+    def _tool(directory, name, source):
+        path = directory / name
+        path.write_text(f"#!{sys.executable}\n" + source)
+        path.chmod(0o755)
+
+    def _provision(self, script, environment):
+        result = subprocess.run(["sh", "-c", script], env=environment, capture_output=True, text=True)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+
+class ContainerdMigrationTests(unittest.TestCase):
+    def _record(self, status="Stopped"):
+        return {"name": "fixture", "status": status, "config": {
+            "cpus": 7, "memory": "11GiB", "mounts": [{"location": "/operator-owned"}],
+            "portForwards": [{"guestPort": 6443, "hostPort": 18448}],
+            "provision": [{"mode": "system", "script": "before\n" + MIGRATION.LEGACY + "after\n"},
+                          {"mode": "user", "script": "echo operator-script\n"}],
+        }}
+
+    def test_migration_preserves_unrelated_configuration_and_is_idempotent(self):
+        record = self._record()
+        original = copy.deepcopy(record)
+        updates = MIGRATION.provision_updates(record["config"])
+        self.assertEqual(len(updates), 1)
+        self.assertEqual(record, original, "planning must not mutate the source")
+        index, script = updates[0]
+        self.assertEqual(index, 0)
+        self.assertEqual(script, "before\n" + MIGRATION.current_block() + "after\n")
+        record["config"]["provision"][index]["script"] = script
+        self.assertEqual(MIGRATION.provision_updates(record["config"]), [])
+
+    def test_unknown_legacy_changes_are_not_overwritten(self):
+        for script in (
+            MIGRATION.LEGACY.replace("systemctl restart", "custom-systemctl restart"),
+            MIGRATION.LEGACY.replace("default >", "default > "),
+        ):
+            with self.subTest(script=script):
+                record = self._record()
+                record["config"]["provision"][0]["script"] = script
+                with self.assertRaisesRegex(ValueError, "operator changes"):
+                    MIGRATION.provision_updates(record["config"])
+
+    def test_running_vm_requires_explicit_maintenance_stop(self):
+        with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run") as run:
+            run.return_value = subprocess.CompletedProcess([], 0, json.dumps(self._record("Running")), "")
+            with self.assertRaisesRegex(RuntimeError, "stop the VM"):
+                MIGRATION.main()
+            self.assertEqual(run.call_count, 1, "must not edit, stop, or start the VM")
+
+    def test_check_does_not_edit_a_running_vm(self):
+        with patch.object(sys, "argv", ["migration", "fixture", "--check"]), patch.object(MIGRATION.subprocess, "run") as run:
+            run.return_value = subprocess.CompletedProcess([], 0, json.dumps(self._record("Running")), "")
+            self.assertEqual(MIGRATION.main(), 0)
+            self.assertEqual(run.call_count, 1)
+
+    def test_stopped_vm_edit_is_verified_by_readback(self):
+        record = self._record()
+        saved = copy.deepcopy(record)
+        saved["config"]["provision"][0]["script"] = MIGRATION.provision_updates(record["config"])[0][1]
+        responses = [subprocess.CompletedProcess([], 0, json.dumps(value), "") for value in (record, {}, saved)]
+        with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run", side_effect=responses) as run:
+            self.assertEqual(MIGRATION.main(), 0)
+            edit = run.call_args_list[1].args[0]
+            self.assertEqual(edit[:4], ["limactl", "edit", "fixture", "--tty=false"])
+            self.assertEqual(edit[4], "--set")
+            self.assertEqual(json.loads(edit[5].split(" = ", 1)[1]), saved["config"]["provision"][0]["script"])
+            self.assertEqual(len(run.call_args_list), 3)
+
+    def test_failed_readback_is_not_reported_as_migrated(self):
+        record = self._record()
+        responses = [subprocess.CompletedProcess([], 0, json.dumps(value), "") for value in (record, {}, record)]
+        with patch.object(sys, "argv", ["migration", "fixture"]), patch.object(MIGRATION.subprocess, "run", side_effect=responses):
+            with self.assertRaisesRegex(RuntimeError, "differs"):
+                MIGRATION.main()
+
+
+if __name__ == "__main__":
+    unittest.main()