feat(colima): Upgrade to colima 0.10.3 and lima 2.2.1

devenv/checks/limaDns.py

@@ -47,6 +47,20 @@ def check() -> tuple[bool, str]:
     return True, ""
 
 
+def _resolver_service() -> str:
+    # VMs created since colima 0.9 come with dnsmasq, and colima makes it
+    # their resolver instead of systemd-resolved. Check that it's installed
+    # rather than running: if dnsmasq died, it's the one to restart.
+    try:
+        proc.run(
+            ("colima", "exec", "--", "systemctl", "cat", "dnsmasq.service"),
+            stdout=True,
+        )
+    except RuntimeError:
+        return "systemd-resolved.service"
+    return "dnsmasq.service"
+
+
 @fixer
 def fix() -> tuple[bool, str]:
     status = colima.start()
@@ -62,13 +76,13 @@ def fix() -> tuple[bool, str]:
                 "sudo",
                 "systemctl",
                 "restart",
-                "systemd-resolved.service",
+                _resolver_service(),
             ),
             stdout=True,
         )
     except RuntimeError as e:
         print(f"""
-failed to restart the vm's resolved:
+failed to restart the vm's dns resolver:
 {e}
 
 we're going to try restarting colima

devenv/lib/colima.py

@@ -56,12 +56,12 @@ def install_shim(binroot: str) -> None:
 
 
 def install_global() -> None:
-    version = "v0.8.1"
+    version = "v0.10.3"
     cfg = {
         "darwin_x86_64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Darwin-x86_64",
-        "darwin_x86_64_sha256": "791330c62c60389f70e5e1c33a56c35502a9e36e544a418daea0273e539acbf4",
+        "darwin_x86_64_sha256": "3082737fe8a98afda11cba7d9a20b6e56fe80c6153464beda04bec630758770b",
         "darwin_arm64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Darwin-arm64",
-        "darwin_arm64_sha256": "c266fcb272b39221ef6152d2093bb02a1ebadc26042233ad359e1ae52d5d5922",
+        "darwin_arm64_sha256": "980ad8bf61a4ca370243f4cb41401a61276dcd2c2502bee7b9b86f9250169f34",
     }
 
     binroot = f"{root}/bin"
@@ -178,7 +178,7 @@ def start(restart: bool = False) -> ColimaStatus:
     memsize_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
 
     args = [
-        "--cpu",
+        "--cpus",
         f"{cpus // 2}",
         "--memory",
         f"{memsize_bytes // (2 * 1024**3)}",
@@ -194,10 +194,17 @@ def start(restart: bool = False) -> ColimaStatus:
             "--verbose",
             # we're overprovisioning a 512 GiB disk here which is fine because
             # the disk doesn't reserve space and is copy-on-write
+            # (VMs created since colima 0.9 keep container data on a separate
+            # disk of this size, which survives `colima delete` without --data)
             "--disk",
             "512",
+            # colima 0.10.2 stopped propagating file events by default;
+            # keep VMs created now behaving like the ones created before
+            "--mount-inotify",
             # this effectively makes the vm's resolvectl status use:
             # DNS Servers: 8.8.8.8 1.1.1.1 192.168.5.2
+            # (VMs created since colima 0.9 use dnsmasq instead, with only
+            # 8.8.8.8 and 1.1.1.1 as upstreams)
             # https://lima-vm.io/docs/config/network/user/
             # 192.168.5.2 is the host, accessible from the vm
             # sometimes using only the host will result in dns breaking

devenv/lib/limactl.py

@@ -10,17 +10,26 @@
 from devenv.lib import archive
 from devenv.lib import proc
 
+# lima >= 2 looks for its guest agents and drivers relative to limactl,
+# in ../share/lima and ../libexec/lima
+_DATA_DIRS = ("share/lima", "libexec/lima")
+
 
 def uninstall(binroot: str) -> None:
-    for d in (f"{home}/.lima",):
+    prefix = os.path.dirname(binroot)
+    for d in (
+        f"{home}/.lima",
+        *(f"{prefix}/{d}" for d in _DATA_DIRS),
+        # lima < 2 kept its templates and guest agents next to limactl
+        f"{binroot}/templates",
+    ):
         shutil.rmtree(d, ignore_errors=True)
 
     for fp in (
         f"{binroot}/lima",
         f"{binroot}/limactl",
         f"{binroot}/lima-guestagent.Linux-aarch64",
         f"{binroot}/lima-guestagent.Linux-x86_64",
-        f"{binroot}/templates/default.yaml",
     ):
         try:
             os.remove(fp)
@@ -32,46 +41,28 @@ def uninstall(binroot: str) -> None:
 
 
 def _install(url: str, sha256: str, into: str) -> None:
+    prefix = os.path.dirname(into)
     os.makedirs(into, exist_ok=True)
     with tempfile.TemporaryDirectory(dir=into) as tmpd:
         archive_file = archive.download(url, sha256, dest=f"{tmpd}/download")
-
-        # the archive from homebrew has a lima/version prefix
-        archive.unpack_strip_n(archive_file, tmpd, n=2)
+        archive.unpack(archive_file, tmpd)
 
         # the archive was atomically placed into tmpd so
         # these are on the same fs and can be atomically moved too
+        for d in _DATA_DIRS:
+            os.makedirs(os.path.dirname(f"{prefix}/{d}"), exist_ok=True)
+            os.replace(f"{tmpd}/{d}", f"{prefix}/{d}")
         os.replace(f"{tmpd}/bin/lima", f"{into}/lima")
         os.replace(f"{tmpd}/bin/limactl", f"{into}/limactl")
-        os.replace(
-            f"{tmpd}/share/lima/lima-guestagent.Linux-aarch64",
-            f"{into}/lima-guestagent.Linux-aarch64",
-        )
-        os.replace(
-            f"{tmpd}/share/lima/lima-guestagent.Linux-x86_64",
-            f"{into}/lima-guestagent.Linux-x86_64",
-        )
-        os.makedirs(f"{into}/templates", exist_ok=True)
-        os.replace(
-            f"{tmpd}/share/lima/templates/default.yaml",
-            f"{into}/templates/default.yaml",
-        )
 
 
 def install_global() -> None:
-    version = "1.0.3"
+    version = "2.2.1"
     cfg = {
-        # upstream github releases aren't built for macOS 14, so for now
-        # we use homebrew binaries from https://formulae.brew.sh/api/formula/lima.json
-        # unfortunately there's no way to view previous versions of a formula
-        # so if we ever need to upgrade to a specific non-latest version we'll
-        # have to build it ourselves
-        # sonoma
-        "darwin_x86_64": "https://ghcr.io/v2/homebrew/core/lima/blobs/sha256:f675abee28d0f10f335f7a04dc6ad3de12434c83c2f0f32c913061204c137a94",
-        "darwin_x86_64_sha256": "f675abee28d0f10f335f7a04dc6ad3de12434c83c2f0f32c913061204c137a94",
-        # arm64_sonoma
-        "darwin_arm64": "https://ghcr.io/v2/homebrew/core/lima/blobs/sha256:8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a",
-        "darwin_arm64_sha256": "8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a",
+        "darwin_x86_64": f"https://github.com/lima-vm/lima/releases/download/v{version}/lima-{version}-Darwin-x86_64.tar.gz",
+        "darwin_x86_64_sha256": "6b776e4bc41f358e6ee01fefcc78e2eb26f11635a0fcff632234af77a566c6c4",
+        "darwin_arm64": f"https://github.com/lima-vm/lima/releases/download/v{version}/lima-{version}-Darwin-arm64.tar.gz",
+        "darwin_arm64_sha256": "9e9eacce88f37e185c346bad73aa6136f738d8cdf8c3bb23cd42b071824bc66e",
     }
 
     binroot = f"{root}/bin"
@@ -82,9 +73,18 @@ def install_global() -> None:
     ):
         stdout = proc.run((f"{binroot}/limactl", "--version"), stdout=True)
         installed_version = stdout.strip().split()[-1]
-        if version == installed_version:
+        # an interrupted install can leave limactl without its guest agents
+        missing = [
+            f"{root}/{d}"
+            for d in _DATA_DIRS
+            if not os.path.isdir(f"{root}/{d}")
+        ]
+        if version != installed_version:
+            print(f"installed limactl {installed_version} is unexpected!")
+        elif missing:
+            print(f"limactl {version} is missing {', '.join(missing)}...")
+        else:
             return
-        print(f"installed limactl {installed_version} is unexpected!")
 
     uninstall(binroot)
     _install(cfg[SYSTEM_MACHINE], cfg[f"{SYSTEM_MACHINE}_sha256"], binroot)

tests/checks/test_limaDns.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from unittest import mock
+
+import pytest
+
+from devenv.checks import limaDns
+from devenv.lib import colima
+
+
+@pytest.mark.parametrize(
+    ("dnsmasq_installed", "dnsmasq_active", "service"),
+    (
+        (True, True, "dnsmasq.service"),
+        # e.g. dnsmasq crashed, which is why DNS is broken
+        (True, False, "dnsmasq.service"),
+        # VMs created before colima 0.9
+        (False, False, "systemd-resolved.service"),
+    ),
+)
+def test_fix_restarts_the_vms_resolver(
+    dnsmasq_installed: bool, dnsmasq_active: bool, service: str
+) -> None:
+    cmds: list[tuple[str, ...]] = []
+
+    def run(cmd: tuple[str, ...], stdout: bool = False) -> str:
+        cmds.append(cmd)
+        vm_cmd = cmd[3:]
+        if vm_cmd == ("systemctl", "cat", "dnsmasq.service"):
+            if not dnsmasq_installed:
+                raise RuntimeError("Command failed! (code 1)")
+        elif vm_cmd[:2] == ("systemctl", "is-active") and not dnsmasq_active:
+            raise RuntimeError("Command failed! (code 3)")
+        return ""
+
+    with (
+        mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP),
+        mock.patch("devenv.checks.limaDns.proc.run", side_effect=run),
+    ):
+        assert limaDns.fix() == (True, "")
+
+    assert cmds[-1] == (
+        "colima",
+        "exec",
+        "--",
+        "sudo",
+        "systemctl",
+        "restart",
+        service,
+    )

tests/lib/test_limactl.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+import tarfile
+from unittest.mock import patch
+
+import pytest
+
+from devenv.lib import limactl
+
+# the layout of upstream's lima-2.x-Darwin-*.tar.gz, abridged
+RELEASE_FILES = (
+    "./bin/lima",
+    "./bin/limactl",
+    "./bin/docker.lima",
+    "./libexec/lima/lima-driver-krunkit",
+    "./share/lima/lima-guestagent.Linux-aarch64.gz",
+    "./share/lima/templates/default.yaml",
+    "./share/doc/lima/README.md",
+)
+
+
+def _release(tmp_path: pathlib.Path) -> str:
+    src = tmp_path / "release"
+    for name in RELEASE_FILES:
+        (src / name).parent.mkdir(parents=True, exist_ok=True)
+        (src / name).write_text(name)
+        if name.startswith("./bin/"):
+            # install_global() only trusts what shutil.which finds
+            (src / name).chmod(0o755)
+    path = f"{tmp_path}/lima.tar.gz"
+    with tarfile.open(path, "w:gz") as tarf:
+        tarf.add(src, arcname=".")
+    return path
+
+
+def _install(tmp_path: pathlib.Path, binroot: str) -> None:
+    release = _release(tmp_path)
+
+    def download(url: str, sha256: str, dest: str) -> str:
+        shutil.copy(release, dest)
+        return dest
+
+    with patch("devenv.lib.archive.download", side_effect=download):
+        limactl._install("url", "sha256", binroot)
+
+
+def test_install_keeps_data_dirs_next_to_bin(tmp_path: pathlib.Path) -> None:
+    prefix = f"{tmp_path}/sentry-devenv"
+    binroot = f"{prefix}/bin"
+
+    _install(tmp_path, binroot)
+
+    # lima >= 2 finds these relative to limactl
+    assert os.path.exists(f"{binroot}/lima")
+    assert os.path.exists(f"{binroot}/limactl")
+    assert os.path.exists(
+        f"{prefix}/share/lima/lima-guestagent.Linux-aarch64.gz"
+    )
+    assert os.path.exists(f"{prefix}/libexec/lima/lima-driver-krunkit")
+    # only lima and limactl go on PATH
+    assert sorted(os.listdir(binroot)) == ["lima", "limactl"]
+
+
+def test_install_replaces_a_previous_install(tmp_path: pathlib.Path) -> None:
+    prefix = f"{tmp_path}/sentry-devenv"
+    binroot = f"{prefix}/bin"
+    _install(tmp_path, binroot)
+    stale = f"{prefix}/share/lima/stale"
+    open(stale, "w").close()
+
+    limactl.uninstall(binroot)
+    _install(tmp_path, binroot)
+
+    assert not os.path.exists(stale)
+    assert os.path.exists(f"{binroot}/limactl")
+
+
+def test_uninstall_removes_the_lima_1_layout(tmp_path: pathlib.Path) -> None:
+    binroot = f"{tmp_path}/sentry-devenv/bin"
+    os.makedirs(f"{binroot}/templates")
+    for name in (
+        "lima",
+        "limactl",
+        "lima-guestagent.Linux-aarch64",
+        "lima-guestagent.Linux-x86_64",
+        "templates/default.yaml",
+        "colima",
+    ):
+        open(f"{binroot}/{name}", "w").close()
+
+    limactl.uninstall(binroot)
+
+    assert os.listdir(binroot) == ["colima"]
+
+
+@pytest.mark.parametrize(
+    ("missing", "reinstalls"),
+    (
+        (None, False),
+        # e.g. an interrupted install
+        ("share/lima", True),
+        ("libexec/lima", True),
+    ),
+)
+def test_install_global_reinstalls_an_incomplete_install(
+    tmp_path: pathlib.Path, missing: str | None, reinstalls: bool
+) -> None:
+    root = f"{tmp_path}/sentry-devenv"
+    binroot = f"{root}/bin"
+    _install(tmp_path, binroot)
+    if missing:
+        shutil.rmtree(f"{root}/{missing}")
+
+    with (
+        patch.object(limactl, "root", root),
+        patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"),
+        patch(
+            "devenv.lib.limactl.proc.run", return_value="limactl version 2.2.1"
+        ),
+        patch.object(limactl, "_install") as install,
+    ):
+        limactl.install_global()
+
+    assert install.called == reinstalls