diff --git a/AGENTS.md b/AGENTS.md index bbe8a993..8f1b9c91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,17 @@ Co-Authored-By: - Run your checkout against another repo: `uv run --project ~/code/devenv devenv sync` from that repo's root - Run the `bootstrap` workflow locally with `gh act`: `make test` +## Cross-repo upgrade checks + +When you change devenv's internal Python pin or its Colima or Lima versions, smoke-test the installed devenv against `~/dev/sentry`. Do this before calling the upgrade complete. + +- From `~/dev/sentry`, run `devenv sync` and check `devenv --version`, `.venv/bin/python --version`, `colima version`, and `limactl --version`. +- Read Sentry's `.python-version`. Devenv's internal Python pin does not set Sentry's virtualenv version. +- Run `devservices up` and confirm `devservices status` reports the needed services as healthy. +- Run a few targeted pytest tests that use those services. Confirm they use a test database, then report the commands and results. +- Do not stop, delete, or prune the developer's default Colima VM. +- Use a disposable profile only after verifying that every VM and data path is isolated. + ## Invariants - Python is pinned to 3.13.16 (`requires-python` in `pyproject.toml`, `.python-version`). diff --git a/README.md b/README.md index 3da90fba..ddc18c37 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,9 @@ If your devenv is old enough that it has no `update` command, run: `devenv colima start` -Starts colima with resources sized for your machine. The default `colima start` may underprovision resources. Run `devenv colima start` on a new machine, and again after `colima delete`. The other subcommands are `restart`, `check`, and `stop`. +Starts or creates devenv's Colima VM with resources sized for your machine. Existing VM settings are kept. The other subcommands are `restart`, `check`, and `stop`. + +On affected Apple M4/M5 systems, this and `devenv colima restart` check whether the VM kernel reports SVE support the CPU does not have. Devenv upgrades the VM kernel when needed, restarts once, and verifies the result. `devenv doctor` checks this too. `devenv pin_gha [file ...]` @@ -268,7 +270,40 @@ def main(context: dict[str, str]) -> int: ### colima -On macOS, `devenv bootstrap` installs colima and the docker CLI globally. If you installed devenv before 1.14.0, run `devenv update` to get them. +`devenv bootstrap` and `devenv update` install Colima `v0.10.3` and Lima `v2.2.1` on macOS. + +#### New macOS installations + +Colima 0.10.3 creates new VMs from its Ubuntu 24.04.4 image, whose kernel is expected to report SVE correctly. M4 Pro validation confirmed this on that host. New VMs use a separate persistent 512 GiB container data disk and Docker's containerd image store. Inotify is off by default. + +#### Existing macOS installations + +Updating devenv keeps the existing VM, Docker images, containers, and named volumes. It does not recreate the VM or convert its disk layout. Colima 0.8.1 VMs keep Docker data on their root disk; newer VMs use a separate persistent data disk. Do not delete a legacy VM as routine cleanup: its containers and named volumes can be lost with its root disk. + +`devenv update` does not stop a running VM. Before the next upgraded start, devenv checks a running daemon through the `colima` Docker context and records its store choice in `~/.colima/default/colima.yaml`. For a stopped VM, it uses Colima's persisted store and Lima disk metadata to identify the layout. If the evidence is ambiguous, startup stops with instructions to set `docker.features.containerd-snapshotter` explicitly. Existing explicit values are preserved, and devenv does not remove Docker data. + +Affected M4/M5 Macs may need an in-VM kernel update because existing VMs keep their original kernel. Devenv installs and boots the update when needed. If a previous repair installed the kernel but did not boot it, run `devenv colima restart`. This does not convert the VM to the new data disk layout. + +Lima 2.1 changed the instance disk layout. After starting a VM with Lima 2.x, Lima 1.x cannot start that instance. Downgrading devenv or its binaries does not restore the old layout. See [Lima's compatibility discussion](https://github.com/lima-vm/lima/discussions/4747). + +#### Check or change the Docker image store + +Run `docker --context=colima info` to see the active storage driver. The classic store reports `overlay2`. The containerd store reports `io.containerd.snapshotter.v1` in the driver status. Use the explicit `colima` context so Docker Desktop's current context does not affect the result. + +If images or containers appear missing after an upgrade, inspect `docker --context=colima info`, `docker --context=colima images`, `docker --context=colima ps -a`, and `docker --context=colima volume ls`. Check `docker.features.containerd-snapshotter` in `~/.colima/default/colima.yaml` against the store containing your data. For a legacy `overlay2` VM, use: + +```yaml +docker: + features: + buildkit: true + containerd-snapshotter: false +``` + +Set it to `true` only after confirming the VM's data is in the containerd store. Start Colima after correcting the value. Switching stores hides images and containers in the other store. Do not delete the VM or its disks to diagnose the problem. + +#### Enable inotify propagation + +To propagate host file events into the VM, set `mountInotify: true` in `~/.colima/default/colima.yaml` or start Colima with `--mount-inotify`. Devenv preserves an existing `true` setting. Inotify is off for new VMs because Colima watches configured mounts, including devenv's writable home mount. Watching a large home directory can increase host CPU and memory use; see [Colima issue 1569](https://github.com/abiosoft/colima/issues/1569). ### gcloud diff --git a/devenv/checks/colimaSve.py b/devenv/checks/colimaSve.py new file mode 100644 index 00000000..472b1404 --- /dev/null +++ b/devenv/checks/colimaSve.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from devenv.lib import colima +from devenv.lib_check.types import checker +from devenv.lib_check.types import fixer + +tags: set[str] = {"builtin", "colima"} +name = "colima's VM shouldn't report SVE support on CPUs without SVE" + + +@checker +def check() -> tuple[bool, str]: + try: + misreported = colima.vm_misreports_sve() + except RuntimeError: + # colima isn't running, which another check covers + return True, "" + + if misreported: + return ( + False, + "colima's VM kernel reports SVE2, but this CPU has no SVE. " + "Programs that use it crash with 'Illegal instruction'.", + ) + + return True, "" + + +@fixer +def fix() -> tuple[bool, str]: + # upgrades the VM's kernel and restarts colima to boot it + status = colima.start(restart=True) + if status != colima.ColimaStatus.UP: + return False, "colima is unhealthy" + + # start only warns if it couldn't upgrade the kernel + try: + misreported = colima.vm_misreports_sve() + except RuntimeError as e: + return False, f"{e}" + if misreported: + return False, "colima's VM still misreports SVE, see above" + + return True, "" diff --git a/devenv/checks/limaDns.py b/devenv/checks/limaDns.py index 3e83ead3..dd703e5f 100644 --- a/devenv/checks/limaDns.py +++ b/devenv/checks/limaDns.py @@ -10,35 +10,49 @@ tags: set[str] = {"builtin", "colima"} name = "colima's DNS isn't working" +_DNS_CHECK = """ +import socket + +try: + socket.getaddrinfo("ghcr.io", None) +except socket.gaierror as e: + raise SystemExit(f"failed to resolve ghcr.io: {e}") +""" + +_DNS_CONFIG = """ +from pathlib import Path + +resolv_conf = Path("/etc/resolv.conf") +print(resolv_conf.resolve()) +print(resolv_conf.read_text()) +""" + + +def _resolver_service(target: str, resolv_conf: str) -> str | None: + if target.startswith("/run/systemd/resolve/"): + return "systemd-resolved.service" + + nameservers = { + line.split()[1] + for line in resolv_conf.splitlines() + if line.strip().startswith("nameserver ") and len(line.split()) == 2 + } + if "127.0.0.53" in nameservers: + return "systemd-resolved.service" + if "# Generated by Colima" in resolv_conf and nameservers: + return "dnsmasq.service" + return None + @checker def check() -> tuple[bool, str]: - # dns resolution can... stop working if colima's running - # and wifi changes to some other network that gives macos some - # weird nameservers - status = colima.check() if status != colima.ColimaStatus.UP: return False, "Colima isn't running." try: proc.run( - ( - "colima", - "exec", - "--", - "python3", - "-Su", - "-c", - """ -import socket - -try: - socket.getaddrinfo("ghcr.io", None) -except socket.gaierror as e: - raise SystemExit(f"failed to resolve ghcr.io: {e}") -""", - ), + ("colima", "exec", "--", "python3", "-Su", "-c", _DNS_CHECK), stdout=True, ) except RuntimeError as e: @@ -50,25 +64,35 @@ def check() -> tuple[bool, str]: @fixer def fix() -> tuple[bool, str]: status = colima.start() - if status == colima.ColimaStatus.UNHEALTHY: - return False, "colima started, but it's unhealthy" + if status != colima.ColimaStatus.UP: + return False, f"Colima is not healthy: {status.name.lower()}" + + try: + config = proc.run( + ("colima", "exec", "--", "python3", "-Su", "-c", _DNS_CONFIG), + stdout=True, + ) + except RuntimeError as e: + return False, f"Unable to inspect Colima's /etc/resolv.conf: {e}" + + target, _, resolv_conf = config.partition("\n") + service = _resolver_service(target, resolv_conf) + if service is None: + return ( + False, + "Unable to identify Colima's DNS resolver from /etc/resolv.conf. " + "Inspect the VM resolver configuration and restart the selected " + "resolver manually.", + ) try: proc.run( - ( - "colima", - "exec", - "--", - "sudo", - "systemctl", - "restart", - "systemd-resolved.service", - ), + ("colima", "exec", "--", "sudo", "systemctl", "restart", service), stdout=True, ) except RuntimeError as e: print(f""" -failed to restart the vm's resolved: +failed to restart the vm's {service}: {e} we're going to try restarting colima @@ -80,4 +104,7 @@ def fix() -> tuple[bool, str]: except RuntimeError as e: return False, f"{e}" + healthy, detail = check() + if not healthy: + return False, f"DNS resolution is still failing after repair: {detail}" return True, "" diff --git a/devenv/colima.py b/devenv/colima.py index bacc6aab..0404a4e8 100644 --- a/devenv/colima.py +++ b/devenv/colima.py @@ -18,17 +18,8 @@ def main(context: Context, argv: Sequence[str] | None = None) -> int: args = parser.parse_args(argv) - # TODO: in addition to returning 1 we should print colima logs - # (and/or send to sentry) - if args.command == "start": status = colima.start() - if status == colima.ColimaStatus.UNHEALTHY: - # https://github.com/abiosoft/colima/issues/949 - print("colima seems unhealthy, we'll try restarting once") - status = colima.restart() - if status != colima.ColimaStatus.UP: - return 1 if status != colima.ColimaStatus.UP: return 1 elif args.command == "restart": diff --git a/devenv/lib/colima.py b/devenv/lib/colima.py index d94e68d7..8996818d 100644 --- a/devenv/lib/colima.py +++ b/devenv/lib/colima.py @@ -1,15 +1,22 @@ from __future__ import annotations +import json import os import platform +import re import shutil +import stat import tempfile +from collections.abc import Collection +from dataclasses import dataclass from enum import Enum +from devenv import constants from devenv.constants import SYSTEM_MACHINE from devenv.constants import home from devenv.constants import root from devenv.lib import archive +from devenv.lib import colima_config from devenv.lib import docker from devenv.lib import fs from devenv.lib import proc @@ -18,25 +25,31 @@ ColimaStatus = Enum("ColimaStatus", ("UP", "DOWN", "UNHEALTHY")) +class _StartMode(Enum): + ENSURE_RUNNING = "ensure-running" + REQUESTED_RESTART = "requested-restart" + HEALTH_RECOVERY = "health-recovery" + + +@dataclass(frozen=True) +class _StoragePolicy: + existing_instance: bool + expected_snapshotter: bool | None = None + + def _install(url: str, sha256: str, into: str) -> None: os.makedirs(into, exist_ok=True) with tempfile.TemporaryDirectory(dir=into) as tmpd: - archive.download(url, sha256, dest=f"{tmpd}/colima") - os.replace(f"{tmpd}/colima", f"{into}/colima-bin") - os.chmod(f"{into}/colima-bin", 0o775) + downloaded = archive.download(url, sha256, dest=f"{tmpd}/colima") + os.chmod(downloaded, 0o775) + os.replace(downloaded, f"{into}/colima-bin") def uninstall(binroot: str) -> None: - for d in (f"{home}/.lima",): - shutil.rmtree(d, ignore_errors=True) - - for fp in (f"{binroot}/colima",): + for fp in (f"{binroot}/colima", f"{binroot}/colima-bin"): try: os.remove(fp) except FileNotFoundError: - # it's better to do this than to guard with - # os.path.exists(fp) because if it's an invalid or circular - # symlink the result'll be False! pass @@ -46,7 +59,6 @@ def install_shim(binroot: str) -> None: """#!/bin/sh export COLIMA_HOME={home}/.colima -# needed to ensure COLIMA_HOME is what we want it to be unset XDG_CONFIG_HOME exec {binroot}/colima-bin "$@" @@ -55,30 +67,44 @@ def install_shim(binroot: str) -> None: ) +def _installed_colima_version(command: str) -> str | None: + try: + stdout = proc.run((command, "--version"), stdout=True) + except RuntimeError as error: + print(f"existing Colima installation is incomplete: {error}") + return None + words = stdout.strip().split() + if not words: + print("existing Colima installation returned no version; reinstalling") + return None + return words[-1] + + def install_global() -> None: - version = "v0.8.1" + version = "v0.10.3" cfg = { "darwin_x86_64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Darwin-x86_64", - "darwin_x86_64_sha256": "791330c62c60389f70e5e1c33a56c35502a9e36e544a418daea0273e539acbf4", + "darwin_x86_64_sha256": "3082737fe8a98afda11cba7d9a20b6e56fe80c6153464beda04bec630758770b", "darwin_arm64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Darwin-arm64", - "darwin_arm64_sha256": "c266fcb272b39221ef6152d2093bb02a1ebadc26042233ad359e1ae52d5d5922", + "darwin_arm64_sha256": "980ad8bf61a4ca370243f4cb41401a61276dcd2c2502bee7b9b86f9250169f34", } binroot = f"{root}/bin" - if shutil.which("colima", path=binroot) == f"{binroot}/colima": - if not os.path.exists(f"{binroot}/colima-bin"): - os.rename(f"{binroot}/colima", f"{binroot}/colima-bin") + colima_path = f"{binroot}/colima" + binary_path = f"{binroot}/colima-bin" + if shutil.which("colima", path=binroot) == colima_path: + installed_version = _installed_colima_version(colima_path) + if not os.path.exists(binary_path) and installed_version is not None: + os.replace(colima_path, binary_path) install_shim(binroot) - - stdout = proc.run((f"{binroot}/colima", "--version"), stdout=True) - installed_version = stdout.strip().split()[-1] + installed_version = _installed_colima_version(colima_path) if version == installed_version: return - print(f"installed colima {installed_version} is unexpected!") + if installed_version is not None: + print(f"installed colima {installed_version} is unexpected!") print(f"installing colima {version}...") - uninstall(binroot) _install(cfg[SYSTEM_MACHINE], cfg[f"{SYSTEM_MACHINE}_sha256"], binroot) install_shim(binroot) @@ -121,8 +147,6 @@ def check() -> ColimaStatus: if not os.path.exists(f"{home}/.colima/default/docker.sock"): return ColimaStatus.DOWN - # if colima's up, we should be able to communicate with that docker socket - # at the most basic level try: proc.run( (docker_executable, "--context=colima", "version"), stdout=True @@ -130,7 +154,6 @@ def check() -> ColimaStatus: except RuntimeError: return ColimaStatus.UNHEALTHY - # https://github.com/abiosoft/colima/issues/949 healthy = docker.check_docker_to_host_connectivity(context="colima") if not healthy: return ColimaStatus.UNHEALTHY @@ -138,83 +161,665 @@ def check() -> ColimaStatus: return ColimaStatus.UP +def _colima_lima_home() -> str: + return constants.user_environ.get("LIMA_HOME") or f"{home}/.colima/_lima" + + +def _instance_path() -> str: + return f"{_colima_lima_home()}/colima" + + +def _config_path() -> str: + return f"{home}/.colima/default/colima.yaml" + + +def _template_path() -> str: + return f"{home}/.colima/_templates/default.yaml" + + +def _read_optional_config(path: str) -> colima_config.ColimaConfig | None: + try: + os.lstat(path) + except FileNotFoundError: + return None + except OSError as error: + raise colima_config.StorageMigrationBlocked( + f"Unable to inspect Colima config {path}: {error}. Inspect the " + "config before starting a new VM." + ) from error + return colima_config.read(path) + + +def _instance_exists() -> bool: + path = _instance_path() + try: + mode = os.lstat(path).st_mode + except FileNotFoundError: + return False + except OSError as error: + raise colima_config.StorageMigrationBlocked( + f"Unable to inspect Colima's Lima instance path {path}: {error}. " + "Inspect the existing VM before starting it." + ) from error + if stat.S_ISLNK(mode): + raise colima_config.StorageMigrationBlocked( + f"Colima's Lima instance path {path} is a symlink; inspect the " + "existing VM location before starting it." + ) + if not stat.S_ISDIR(mode): + raise colima_config.StorageMigrationBlocked( + f"Colima's Lima instance path {path} is not a directory; inspect " + "the existing VM before starting it." + ) + return True + + +def _docker_store_from_info() -> bool: + try: + output = proc.run( + ( + "docker", + "--context=colima", + "info", + "--format", + "{{.Driver}}|{{json .DriverStatus}}", + ), + stdout=True, + ) + except RuntimeError as error: + raise colima_config.StorageMigrationBlocked( + "The existing Colima VM is running, but Docker's storage backend " + f"could not be verified through the `colima` context: {error}. " + "No backend setting was changed. Restore Docker connectivity, " + "then run `devenv colima start` again." + ) from error + + driver, separator, details = output.partition("|") + if not separator: + raise colima_config.StorageMigrationBlocked( + "Docker returned an unrecognized `info` response for the `colima` " + "context. No backend setting was changed. Inspect `docker " + "--context=colima info` before starting Colima." + ) + try: + driver_status = json.loads(details) + except json.JSONDecodeError as error: + if driver.strip() == "overlay2": + return False + raise colima_config.StorageMigrationBlocked( + "Docker returned invalid storage-driver details for the `colima` " + "context. No backend setting was changed. Inspect `docker " + "--context=colima info` before starting Colima." + ) from error + if isinstance(driver_status, list) and any( + isinstance(entry, list) + and len(entry) == 2 + and entry[0] == "driver-type" + and entry[1] == "io.containerd.snapshotter.v1" + for entry in driver_status + ): + return True + if driver.strip() == "overlay2": + return False + if not isinstance(driver_status, list): + raise colima_config.StorageMigrationBlocked( + "Docker returned an unrecognized storage-driver response for the " + "`colima` context. No backend setting was changed. Inspect " + "`docker --context=colima info` before starting Colima." + ) + raise colima_config.StorageMigrationBlocked( + f"Docker reports storage driver `{driver.strip()}`, which devenv " + "cannot safely classify. No backend setting was changed. Inspect " + "`docker --context=colima info` before starting Colima." + ) + + +def _legacy_root_disk(config: colima_config.ColimaConfig) -> bool: + store_path = f"{home}/.colima/_store/colima.json" + store: dict[str, object] | None = None + try: + store_mode = os.lstat(store_path).st_mode + except FileNotFoundError: + store_mode = None + except OSError as error: + raise colima_config.StorageMigrationBlocked( + f"Unable to inspect Colima store metadata {store_path}: {error}. " + "Inspect the existing VM before starting it." + ) from error + if store_mode is not None and stat.S_ISLNK(store_mode): + raise colima_config.StorageMigrationBlocked( + f"Colima store metadata {store_path} is a symlink; inspect it " + "before starting the VM." + ) + if store_mode is not None: + if not stat.S_ISREG(store_mode): + raise colima_config.StorageMigrationBlocked( + f"Colima store metadata {store_path} is not a regular file; " + "inspect the existing VM before starting it." + ) + try: + with open(store_path, encoding="utf-8") as store_file: + parsed = json.load(store_file) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise colima_config.StorageMigrationBlocked( + f"Unable to read Colima store metadata {store_path}: {error}. " + "Inspect the existing VM before starting it." + ) from error + if not isinstance(parsed, dict): + raise colima_config.StorageMigrationBlocked( + f"Colima store metadata {store_path} has an unsupported " + "format; inspect the existing VM before starting it." + ) + store = parsed + + if store is not None: + formatted = store.get("disk_formatted") + runtime = store.get("disk_runtime") + if not isinstance(formatted, bool) or not isinstance(runtime, str): + raise colima_config.StorageMigrationBlocked( + f"Colima store metadata {store_path} has unknown disk fields; " + "inspect the existing VM before starting it." + ) + if formatted or runtime: + return False + + limactl_path = f"{root}/bin/limactl" + if not os.path.isfile(limactl_path): + raise colima_config.StorageMigrationBlocked( + f"Cannot verify whether Colima has a persistent data disk because " + f"`{limactl_path}` is missing. Finish `devenv update` and retry." + ) + try: + disk_info = proc.run( + (limactl_path, "disk", "list", "--json", "colima"), + stdout=True, + env={"LIMA_HOME": _colima_lima_home()}, + ) + except RuntimeError as error: + raise colima_config.StorageMigrationBlocked( + "Cannot verify whether the existing Colima VM has a persistent " + f"data disk: {error}. No backend setting was changed." + ) from error + if disk_info.strip(): + try: + disk = json.loads(disk_info) + except json.JSONDecodeError as error: + raise colima_config.StorageMigrationBlocked( + "Lima returned invalid disk metadata for Colima. No backend " + "setting was changed; repair the Lima installation and retry." + ) from error + else: + disk = None + if disk is not None and not isinstance(disk, dict): + raise colima_config.StorageMigrationBlocked( + "Lima returned an unsupported disk metadata format for Colima. " + "No backend setting was changed." + ) + if isinstance(disk, dict): + disk_name = disk.get("name") + if disk_name == "colima": + return False + if disk_name not in {None, ""}: + raise colima_config.StorageMigrationBlocked( + f"Lima reports unexpected disk `{disk_name}` for Colima. " + "No backend setting was changed." + ) + + return ( + not config.root_disk_configured + and config.disk_gib == 512 + and config.runtime in {None, "docker"} + ) + + +def _prepare_docker_store(status: ColimaStatus) -> _StoragePolicy: + if not _instance_exists(): + config_path = _config_path() + profile_config = _read_optional_config(config_path) + config = profile_config + template = None + if config is None or config.runtime is None: + template = _read_optional_config(_template_path()) + if template is not None: + config = template + if config is None: + return _StoragePolicy( + existing_instance=False, expected_snapshotter=True + ) + if ( + config is profile_config + and profile_config is not None + and profile_config.runtime is None + and template is None + and profile_config.snapshotter is not None + ): + raise colima_config.StorageMigrationBlocked( + f"Colima profile {config_path} has an explicit Docker storage " + "choice but no runtime. Colima treats this profile as empty " + "and will ignore that choice. Set `runtime: docker` before " + "starting the VM." + ) + if config.runtime not in {None, "docker"}: + return _StoragePolicy(existing_instance=False) + if ( + config.snapshotter is None + and config is profile_config + and config.runtime + ): + _persist_docker_store(config_path, True) + if config.snapshotter is None: + return _StoragePolicy( + existing_instance=False, expected_snapshotter=True + ) + return _StoragePolicy( + existing_instance=False, expected_snapshotter=config.snapshotter + ) + + config_path = _config_path() + config = colima_config.read(config_path) + if config.runtime not in {None, "docker"}: + return _StoragePolicy(existing_instance=True) + if config.snapshotter is not None: + return _StoragePolicy( + existing_instance=True, expected_snapshotter=config.snapshotter + ) + + if status == ColimaStatus.DOWN: + overlay2 = _legacy_root_disk(config) + if not overlay2: + raise colima_config.StorageMigrationBlocked( + "The stopped Colima VM has no explicit Docker storage choice, " + "and its data-disk metadata does not prove which image store " + "contains its images and containers. Colima 0.10.3 could hide " + "them on the next start. Inspect the existing backend, then " + "set `docker.features.containerd-snapshotter: false` for " + "overlay2 or `true` for containerd in " + f"`{config_path}`. Do not delete the VM or its disks." + ) + selected_store = False + else: + try: + selected_store = _docker_store_from_info() + except colima_config.StorageMigrationBlocked as error: + if status != ColimaStatus.UNHEALTHY: + raise + try: + overlay2 = _legacy_root_disk(config) + except colima_config.StorageMigrationBlocked as metadata_error: + raise colima_config.StorageMigrationBlocked( + f"{error} Saved Colima metadata could not establish an " + f"overlay2 legacy VM: {metadata_error}" + ) from metadata_error + if not overlay2: + raise colima_config.StorageMigrationBlocked( + f"{error} Saved Colima metadata does not positively " + "identify an overlay2 legacy VM. Inspect the existing " + "Docker store before starting it." + ) from error + selected_store = False + + _persist_docker_store(config_path, selected_store) + return _StoragePolicy( + existing_instance=True, expected_snapshotter=selected_store + ) + + +def _persist_docker_store(config_path: str, snapshotter: bool) -> None: + changed = colima_config.set_snapshotter(config_path, snapshotter) + updated_config = colima_config.read(config_path) + if updated_config.snapshotter != snapshotter: + raise colima_config.StorageMigrationBlocked( + f"Colima config {config_path} changed while its Docker store was " + "being preserved. Inspect the config and Docker backend before " + "starting Colima." + ) + if changed: + store = "containerd" if snapshotter else "overlay2" + print(f"Configured Colima's Docker {store} image store.") + + +def _persist_new_docker_store( + status: ColimaStatus, storage_policy: _StoragePolicy +) -> ColimaStatus: + snapshotter = storage_policy.expected_snapshotter + if ( + status != ColimaStatus.UP + or storage_policy.existing_instance + or snapshotter is None + ): + return status + + try: + config_path = _config_path() + config = colima_config.read(config_path) + if config.runtime not in {None, "docker"}: + raise colima_config.StorageMigrationBlocked( + f"Colima saved runtime `{config.runtime}` for a new VM, but " + "devenv requires Docker storage. Inspect the runtime config " + "before restarting Colima." + ) + if config.snapshotter is not None and config.snapshotter != snapshotter: + raise colima_config.StorageMigrationBlocked( + f"Colima config {config_path} selected a Docker image store " + "that differs from the choice made before startup. Inspect " + "the live Docker backend before restarting Colima." + ) + if config.snapshotter is None: + _persist_docker_store(config_path, snapshotter) + except colima_config.StorageMigrationBlocked as error: + print( + "ERROR: Colima started, but devenv could not persist its Docker " + f"image-store choice: {error}" + ) + return ColimaStatus.UNHEALTHY + + return status + + +def _verify_storage_after_start( + status: ColimaStatus, + storage_policy: _StoragePolicy, + *, + vm_was_started: bool, +) -> ColimaStatus: + if ( + status != ColimaStatus.UP + or not vm_was_started + or storage_policy.expected_snapshotter is None + ): + return status + + try: + actual = _docker_store_from_info() + except colima_config.StorageMigrationBlocked as error: + print(f"ERROR: Cannot verify Colima's Docker image store: {error}") + return ColimaStatus.UNHEALTHY + + if actual != storage_policy.expected_snapshotter: + expected_store = ( + "containerd" if storage_policy.expected_snapshotter else "overlay2" + ) + actual_store = "containerd" if actual else "overlay2" + print( + f"ERROR: Colima started with Docker's {actual_store} image store, " + f"but devenv preserved {expected_store}. Images and containers " + "may be hidden. Stop Colima and inspect its Docker data before " + "changing `containerd-snapshotter`; do not delete the VM or disks." + ) + return ColimaStatus.UNHEALTHY + + return status + + +_APT_GET = ( + "sudo", + "env", + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "-q", + "-o", + "DPkg::Lock::Timeout=120", +) + + +def _vm_run(*cmd: str) -> str: + return proc.run( + ("colima", "ssh", "--", *cmd), pathprepend=f"{root}/bin", stdout=True + ) + + +def _vm_exec(*cmd: str) -> None: + proc.run(("colima", "ssh", "--", *cmd), pathprepend=f"{root}/bin") + + +def _host_has_sme() -> bool: + if not (constants.DARWIN and constants.MACHINE == "arm64"): + return False + try: + out = proc.run( + ("/usr/sbin/sysctl", "-n", "hw.optional.arm.FEAT_SME"), stdout=True + ) + except RuntimeError: + return False + return out == "1" + + +def _misreports_sve(features: Collection[str]) -> bool: + return "sve" not in features and any(f.startswith("sve") for f in features) + + +def vm_misreports_sve() -> bool: + if not _host_has_sme(): + return False + out = _vm_run("grep", "-m1", "^Features", "/proc/cpuinfo") + return _misreports_sve(out.partition(":")[2].split()) + + +def _kernel_version_key(version: str) -> tuple[int, ...]: + return tuple(int(n) for n in re.findall(r"\d+", version)) + + +def _vm_newest_kernel() -> str: + versions = _vm_run("linux-version", "list").split() + if not versions: + raise RuntimeError("found no kernels in colima's VM") + return max(versions, key=_kernel_version_key) + + +def _upgrade_kernel_if_sve_misreported(*, restarting: bool) -> bool: + try: + if not vm_misreports_sve(): + return False + running = _vm_run("uname", "-r") + newest = _vm_newest_kernel() + except RuntimeError as e: + print(f"WARNING: failed to check colima's VM kernel: {e}") + return False + + if newest != running: + if restarting: + return True + print(f""" +WARNING: colima's VM is running Linux {running}, which misreports SVE +support on this CPU, but Linux {newest} is already installed. +Run `devenv colima restart` to boot it. If the VM still runs +Linux {running} after that, ask for help in #discuss-dev-infra. +""") + return False + + print(f""" +colima's VM is running Linux {running}, which tells programs that this Mac's +CPU supports SVE2. Apple M4 and newer don't have SVE, so anything that +believes it (e.g. OpenSSL 4 in Python's cryptography >= 47, or numba) crashes +with "Illegal instruction". + +Installing a newer kernel in the VM, then restarting colima once to boot it. +Your containers and volumes are kept. +""") + try: + _vm_exec(*_APT_GET, "update") + _vm_exec(*_APT_GET, "install", "-y", "linux-image-virtual") + _vm_exec("sync") + newest = _vm_newest_kernel() + except RuntimeError as e: + print(f""" +WARNING: failed to upgrade colima's VM kernel: {e} +`devenv colima start` will try again next time. +""") + return False + + if newest == running: + print(f""" +WARNING: no kernel newer than Linux {running} is available to colima's VM. +Ask for help in #discuss-dev-infra. +""") + return False + + print(f"Installed Linux {newest}, restarting colima to boot it.") + return True + + def start(restart: bool = False) -> ColimaStatus: status = check() + try: + storage_policy = _prepare_docker_store(status) + except colima_config.StorageMigrationBlocked as error: + print(f"ERROR: Cannot safely start Colima: {error}") + return ColimaStatus.UNHEALTHY - if status == ColimaStatus.UP: - if not restart: - # an interrupted start may have left docker on the default context - proc.run(("docker", "context", "use", "colima")) - return ColimaStatus.UP + if status == ColimaStatus.UP and restart: + boot_kernel = _upgrade_kernel_if_sve_misreported(restarting=True) + status = _start_with_health_recovery( + _StartMode.REQUESTED_RESTART, + status, + storage_policy.existing_instance, + ) + status = _verify_storage_after_start( + status, storage_policy, vm_was_started=True + ) + if boot_kernel: + return _report_booted_kernel(status) + return status + + vm_was_started = status != ColimaStatus.UP + if status == ColimaStatus.UNHEALTHY: + print("colima seems unhealthy, attempting one recovery restart") + status = _start( + _StartMode.HEALTH_RECOVERY, + status, + existing_instance=storage_policy.existing_instance, + ) + else: + status = _start_with_health_recovery( + _StartMode.ENSURE_RUNNING, status, storage_policy.existing_instance + ) + + vm_was_started = vm_was_started or status != ColimaStatus.UP + status = _persist_new_docker_store(status, storage_policy) + status = _verify_storage_after_start( + status, storage_policy, vm_was_started=vm_was_started + ) + if status != ColimaStatus.UP: + return status + + boot_kernel = _upgrade_kernel_if_sve_misreported(restarting=restart) + if not boot_kernel: + return status + + status = _start_with_health_recovery( + _StartMode.REQUESTED_RESTART, status, existing_instance=True + ) + status = _verify_storage_after_start( + status, storage_policy, vm_was_started=True + ) + return _report_booted_kernel(status) + + +def _start_with_health_recovery( + mode: _StartMode, status: ColimaStatus, existing_instance: bool +) -> ColimaStatus: + status = _start(mode, status, existing_instance=existing_instance) + if status != ColimaStatus.UNHEALTHY or mode == _StartMode.HEALTH_RECOVERY: + return status + + print("colima seems unhealthy, attempting one recovery restart") + return _start(_StartMode.HEALTH_RECOVERY, existing_instance=True) + + +def _report_booted_kernel(status: ColimaStatus) -> ColimaStatus: + if status != ColimaStatus.UP: + print(""" +WARNING: colima isn't healthy after restarting to boot the VM's upgraded +kernel. Ask for help in #discuss-dev-infra. +""") + return status - # switch to default context to avoid broken contexts (e.g., desktop-linux) - # only once we're actually (re)starting, so that a healthy colima is left - # alone and docker isn't left pointing at the default context try: - proc.run(("docker", "context", "use", "default")) + running = _vm_run("uname", "-r") + still_misreported = vm_misreports_sve() except RuntimeError as e: - print(f"Failed to switch docker context: {e}") + print(f"WARNING: failed to check colima's VM kernel: {e}") + return status + + if still_misreported: + print(f""" +WARNING: colima's VM still misreports SVE support after booting +Linux {running}. Ask for help in #discuss-dev-infra. +""") + else: + print(f"colima's VM is now running Linux {running}.") + + return status + + +def _start( + mode: _StartMode, + status: ColimaStatus | None = None, + *, + existing_instance: bool = False, +) -> ColimaStatus: + if status is None: + status = check() + + if status == ColimaStatus.UP and mode != _StartMode.REQUESTED_RESTART: + proc.run(("docker", "context", "use", "colima")) + return status - # always stop first (colima's internal state can get out of sync) try: - proc.run(("colima", "stop", "-f"), pathprepend=f"{root}/bin") - except RuntimeError as e: - print(f"Failed to stop colima: {e}") + proc.run(("docker", "context", "use", "default")) + except RuntimeError as error: + print(f"Failed to switch docker context: {error}") + + if status == ColimaStatus.UP: + try: + proc.run(("colima", "stop"), pathprepend=f"{root}/bin") + except RuntimeError as error: + raise SystemExit( + f"Failed to stop healthy Colima gracefully: {error}. " + "Resolve the stop failure before retrying." + ) from error + elif status == ColimaStatus.UNHEALTHY: + try: + proc.run(("colima", "stop", "-f"), pathprepend=f"{root}/bin") + except RuntimeError as error: + print(f"Failed to force stop unhealthy Colima: {error}") - # colima start will only WARN if rosetta is unavailable and keep going without it, - # so we need to ensure it's installed and running ourselves rosetta.ensure() cpus = os.cpu_count() if cpus is None: raise SystemExit("failed to determine cpu count") - # SC_PAGE_SIZE is POSIX 2008 - # SC_PHYS_PAGES is a linux addition but also supported by more recent MacOS versions - SC_PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") - SC_PHYS_PAGES = os.sysconf("SC_PHYS_PAGES") - if SC_PAGE_SIZE == -1 or SC_PHYS_PAGES == -1: + page_size = os.sysconf("SC_PAGE_SIZE") + physical_pages = os.sysconf("SC_PHYS_PAGES") + if page_size == -1 or physical_pages == -1: raise SystemExit("failed to determine memsize_bytes") - memsize_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") - - args = [ - "--cpu", - f"{cpus // 2}", - "--memory", - f"{memsize_bytes // (2 * 1024**3)}", - ] - if platform.machine() == "arm64": - args = [*args, "--vm-type=vz", "--vz-rosetta", "--mount-type=virtiofs"] - - proc.run( - ( - # we share the "default" machine across repositories - "colima", - "start", - "--verbose", - # we're overprovisioning a 512 GiB disk here which is fine because - # the disk doesn't reserve space and is copy-on-write - "--disk", - "512", - # this effectively makes the vm's resolvectl status use: - # DNS Servers: 8.8.8.8 1.1.1.1 192.168.5.2 - # https://lima-vm.io/docs/config/network/user/ - # 192.168.5.2 is the host, accessible from the vm - # sometimes using only the host will result in dns breaking - # for any number of reasons (public wifi that gives you some weird dns server, - # tethering, vpn, what have you) - "--dns", - "8.8.8.8", - "--dns", - "1.1.1.1", - # ideally we keep ~ ro, but currently the "default" vm - # is shared across repositories, so for ease of use we'll let home rw - f"--mount=/var/folders:w,/private/tmp/colima:w,{home}:w,/tmp/sentry-profiles:w", - # note: it is not allowed by lima to add top-level root directories like /tmp! - *args, - ), - pathprepend=f"{root}/bin", - ) + memsize_bytes = page_size * physical_pages + + args = ["--verbose"] + if not existing_instance: + args.extend( + ( + "--save-config", + "--cpus", + f"{cpus // 2}", + "--memory", + f"{memsize_bytes // (2 * 1024**3)}", + "--disk", + "512", + "--dns", + "8.8.8.8", + "--dns", + "1.1.1.1", + f"--mount=/var/folders:w,/private/tmp/colima:w,{home}:w,/tmp/sentry-profiles:w", + ) + ) + if platform.machine() == "arm64": + args.extend( + ("--vm-type=vz", "--vz-rosetta", "--mount-type=virtiofs") + ) + + proc.run(("colima", "start", *args), pathprepend=f"{root}/bin") proc.run(("docker", "context", "use", "colima")) diff --git a/devenv/lib/colima_config.py b/devenv/lib/colima_config.py new file mode 100644 index 00000000..0545e29e --- /dev/null +++ b/devenv/lib/colima_config.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import os +import re +import stat +import tempfile +from dataclasses import dataclass + + +class StorageMigrationBlocked(RuntimeError): + pass + + +@dataclass(frozen=True) +class ColimaConfig: + text: str + lines: tuple[str, ...] + docker_line: int | None + docker_end: int | None + docker_inline_empty: bool + features_line: int | None + features_end: int | None + features_inline_empty: bool + snapshotter: bool | None + has_buildkit: bool + root_disk_configured: bool + disk_gib: int | None + runtime: str | None + + +_MAPPING = re.compile(r"^(?P *)(?P[A-Za-z0-9_-]+):(?P.*)$") + + +def _line_text(line: str) -> str: + return line.rstrip("\r\n") + + +def _commentless(value: str) -> str: + return value.split(" #", 1)[0].strip() + + +def _mapping(line: str, indent: int) -> tuple[str, str] | None: + match = _MAPPING.match(_line_text(line)) + if match is None or len(match.group("indent")) != indent: + return None + return match.group("key"), _commentless(match.group("rest")) + + +def _quoted_key(line: str, indent: int) -> bool: + text = _line_text(line) + return ( + re.match(rf"^ {{{indent}}}[\"'][A-Za-z0-9_-]+[\"']:", text) is not None + ) + + +def _block_end(lines: tuple[str, ...], start: int, parent_indent: int) -> int: + for index in range(start + 1, len(lines)): + text = _line_text(lines[index]) + if not text.strip() or text.lstrip().startswith("#"): + continue + if "\t" in text[: len(text) - len(text.lstrip())]: + raise StorageMigrationBlocked( + "Colima's YAML uses tab indentation; edit the Docker features " + "manually and set `containerd-snapshotter: false` for an " + "overlay2 VM." + ) + if not text.startswith(" "): + return index + if len(text) - len(text.lstrip()) <= parent_indent: + return index + return len(lines) + + +def read(path: str) -> ColimaConfig: + try: + mode = os.lstat(path).st_mode + except OSError as error: + raise StorageMigrationBlocked( + f"Unable to inspect Colima config {path}: {error}. Inspect the " + "config and set `docker.features.containerd-snapshotter` " + "explicitly before starting Colima." + ) from error + if stat.S_ISLNK(mode): + raise StorageMigrationBlocked( + f"Colima config {path} is a symlink; devenv will not rewrite it. " + "Inspect it and set `docker.features.containerd-snapshotter` " + "explicitly before starting Colima." + ) + if not stat.S_ISREG(mode): + raise StorageMigrationBlocked( + f"Colima config {path} is not a regular file; devenv will not " + "rewrite it. Inspect it and set " + "`docker.features.containerd-snapshotter` explicitly before " + "starting Colima." + ) + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen( + descriptor, "r", encoding="utf-8", newline="" + ) as config_file: + text = config_file.read() + except (OSError, UnicodeError) as error: + raise StorageMigrationBlocked( + f"Unable to read Colima config {path}: {error}. Inspect the " + "config and set `docker.features.containerd-snapshotter` " + "explicitly before starting Colima." + ) from error + + lines = tuple(text.splitlines(keepends=True)) + if text and not lines: + lines = (text,) + if text.startswith("\ufeff"): + raise StorageMigrationBlocked( + f"Colima config {path} has a UTF-8 byte-order mark; devenv will " + "not rewrite it. Set `docker.features.containerd-snapshotter` " + "explicitly by hand." + ) + + top: dict[str, tuple[int, str]] = {} + for index, line in enumerate(lines): + content = _line_text(line) + if not content.strip() or content.lstrip().startswith("#"): + continue + if content.startswith("\t"): + raise StorageMigrationBlocked( + f"Colima config {path} uses tab indentation; set " + "`docker.features.containerd-snapshotter` manually." + ) + entry = _mapping(line, 0) + if entry is None and re.match( + r'^["\'](?:docker|rootDisk|runtime|disk)["\']:', content + ): + raise StorageMigrationBlocked( + f"Colima config {path} uses a quoted storage key; devenv " + "will not rewrite it. Set `docker.features.containerd-" + "snapshotter` explicitly." + ) + if entry is None and not content.startswith(" "): + quoted_mapping = re.match(r"^(?:'[^']+'|\"[^\"]+\")\s*:", content) + if content.startswith(("{", "[")) or quoted_mapping is None: + raise StorageMigrationBlocked( + f"Colima config {path} uses unsupported root YAML; " + "devenv will not rewrite it. Set " + "`docker.features.containerd-snapshotter` explicitly." + ) + if entry is None and re.match( + r"^(?:docker|rootDisk|runtime|disk)(?:\s|$)", content + ): + raise StorageMigrationBlocked( + f"Colima config {path} has a malformed storage key; devenv " + "will not rewrite it. Set `docker.features.containerd-" + "snapshotter` explicitly." + ) + if entry is None: + continue + key, value = entry + if key in {"docker", "rootDisk", "runtime", "disk"}: + if key in top: + raise StorageMigrationBlocked( + f"Colima config {path} has duplicate `{key}` keys; " + "devenv will not rewrite it. Resolve the duplicate and " + "set `docker.features.containerd-snapshotter` explicitly." + ) + top[key] = (index, value) + + docker = top.get("docker") + docker_line = docker[0] if docker else None + docker_value = docker[1] if docker else "" + docker_inline_empty = docker_value == "{}" + docker_end = ( + _block_end(lines, docker_line, 0) + if docker_line is not None and not docker_inline_empty + else docker_line + 1 + if docker_line is not None + else None + ) + if docker is not None and docker_value not in {"", "{}"}: + raise StorageMigrationBlocked( + f"Colima config {path} has a nonstandard inline `docker` value; " + "devenv will not rewrite it. Set `docker.features.containerd-" + "snapshotter` explicitly in a block mapping." + ) + + features_line: int | None = None + features_end: int | None = None + features_inline_empty = False + snapshotter: bool | None = None + has_buildkit = False + if docker_line is not None and not docker_inline_empty: + direct: dict[str, tuple[int, str]] = {} + for index in range(docker_line + 1, docker_end or docker_line + 1): + content = _line_text(lines[index]) + if not content.strip() or content.lstrip().startswith("#"): + continue + entry = _mapping(lines[index], 2) + if entry is None: + if _quoted_key(lines[index], 2): + raise StorageMigrationBlocked( + f"Colima config {path} uses a quoted Docker key; " + "devenv will not rewrite it. Set " + "`docker.features.containerd-snapshotter` explicitly." + ) + if len(content) - len(content.lstrip()) == 2: + raise StorageMigrationBlocked( + f"Colima config {path} has unsupported Docker YAML; " + "devenv will not rewrite it. Set " + "`docker.features.containerd-snapshotter` explicitly." + ) + continue + key, value = entry + if key in direct: + raise StorageMigrationBlocked( + f"Colima config {path} has duplicate Docker `{key}` keys; " + "devenv will not rewrite it. Resolve the duplicate and " + "set `docker.features.containerd-snapshotter` explicitly." + ) + direct[key] = (index, value) + + features = direct.get("features") + if features is not None: + features_line, features_value = features + features_inline_empty = features_value == "{}" + if features_value not in {"", "{}"}: + raise StorageMigrationBlocked( + f"Colima config {path} has an unsupported inline " + "`docker.features` mapping; set " + "`docker.features.containerd-snapshotter` manually." + ) + features_end = ( + _block_end(lines, features_line, 2) + if not features_inline_empty + else features_line + 1 + ) + nested: dict[str, tuple[int, str]] = {} + if not features_inline_empty: + for index in range(features_line + 1, features_end): + content = _line_text(lines[index]) + if not content.strip() or content.lstrip().startswith("#"): + continue + entry = _mapping(lines[index], 4) + if entry is None: + raise StorageMigrationBlocked( + f"Colima config {path} has unsupported Docker " + "feature YAML; set " + "`docker.features.containerd-snapshotter` " + "explicitly." + ) + key, value = entry + if key in nested: + raise StorageMigrationBlocked( + f"Colima config {path} has duplicate Docker " + f"feature `{key}` keys; devenv will not rewrite it." + ) + nested[key] = (index, value) + + has_buildkit = "buildkit" in nested + configured = nested.get("containerd-snapshotter") + if configured is not None: + value = configured[1] + if value not in {"true", "false"}: + raise StorageMigrationBlocked( + f"Colima config {path} has a non-boolean " + "`containerd-snapshotter` value; choose `true` or " + "`false` explicitly." + ) + snapshotter = value == "true" + + runtime = top.get("runtime") + runtime_value = runtime[1] if runtime else None + if runtime is not None and ( + not runtime_value or not re.fullmatch(r"[A-Za-z0-9_-]+", runtime_value) + ): + raise StorageMigrationBlocked( + f"Colima config {path} has a nonstandard runtime value; inspect " + "it and configure Docker storage explicitly." + ) + + disk = top.get("disk") + disk_value = disk[1] if disk else "" + disk_gib: int | None = None + if disk_value: + try: + disk_gib = int(disk_value) + except ValueError: + raise StorageMigrationBlocked( + f"Colima config {path} has a nonstandard disk size; inspect " + "it and configure Docker storage explicitly." + ) from None + + return ColimaConfig( + text=text, + lines=lines, + docker_line=docker_line, + docker_end=docker_end, + docker_inline_empty=docker_inline_empty, + features_line=features_line, + features_end=features_end, + features_inline_empty=features_inline_empty, + snapshotter=snapshotter, + has_buildkit=has_buildkit, + root_disk_configured="rootDisk" in top, + disk_gib=disk_gib, + runtime=runtime_value, + ) + + +def _newline(config: ColimaConfig) -> str: + for line in config.lines: + if line.endswith("\r\n"): + return "\r\n" + if line.endswith("\n"): + return "\n" + return "\n" + + +def _insert_lines( + lines: list[str], insertion: int, entries: list[str], newline: str +) -> None: + if insertion and not lines[insertion - 1].endswith(("\n", "\r")): + lines[insertion - 1] = f"{lines[insertion - 1]}{newline}" + lines[insertion:insertion] = [f"{entry}{newline}" for entry in entries] + + +def _replacement(config: ColimaConfig, snapshotter: bool) -> str: + if config.snapshotter is not None: + return config.text + + newline = _newline(config) + lines = list(config.lines) + features = config.features_line + if features is not None and config.features_inline_empty: + original = _line_text(lines[features]) + prefix, separator, comment = original.partition(" #") + lines[features] = f" features:{separator}{comment}{newline}" + insertion = features + 1 + entries = [] + if not config.has_buildkit: + entries.append(" buildkit: true") + entries.append( + f" containerd-snapshotter: {str(snapshotter).lower()}" + ) + _insert_lines(lines, insertion, entries, newline) + return "".join(lines) + + if features is not None: + insertion = config.features_end or features + 1 + entries = [] + if not config.has_buildkit: + entries.append(" buildkit: true") + entries.append( + f" containerd-snapshotter: {str(snapshotter).lower()}" + ) + _insert_lines(lines, insertion, entries, newline) + return "".join(lines) + + entries = [ + " features:", + " buildkit: true", + f" containerd-snapshotter: {str(snapshotter).lower()}", + ] + if config.docker_line is None: + if lines and not lines[-1].endswith(("\n", "\r")): + lines[-1] = f"{lines[-1]}{newline}" + if lines and _line_text(lines[-1]).strip(): + lines.append(newline) + lines.extend(f"{entry}{newline}" for entry in ("docker:", *entries)) + return "".join(lines) + + if config.docker_inline_empty: + original = _line_text(lines[config.docker_line]) + prefix, separator, comment = original.partition(" #") + lines[config.docker_line] = f"docker:{separator}{comment}{newline}" + insertion = config.docker_line + 1 + else: + insertion = config.docker_end or config.docker_line + 1 + _insert_lines(lines, insertion, entries, newline) + return "".join(lines) + + +def set_snapshotter(path: str, snapshotter: bool) -> bool: + config = read(path) + replacement = _replacement(config, snapshotter) + if replacement == config.text: + return False + + try: + metadata = os.stat(path, follow_symlinks=False) + except OSError as error: + raise StorageMigrationBlocked( + f"Unable to inspect Colima config {path}: {error}. Set " + "`docker.features.containerd-snapshotter` manually before " + "starting Colima." + ) from error + + temporary_path = "" + try: + descriptor, temporary_path = tempfile.mkstemp( + prefix=".colima.yaml.", dir=os.path.dirname(path) + ) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as file: + os.fchmod(file.fileno(), stat.S_IMODE(metadata.st_mode)) + file.write(replacement) + file.flush() + os.fsync(file.fileno()) + os.replace(temporary_path, path) + temporary_path = "" + except OSError as error: + raise StorageMigrationBlocked( + f"Unable to update Colima config {path}: {error}. The original " + "file was left in place; set " + "`docker.features.containerd-snapshotter` manually before " + "starting Colima." + ) from error + finally: + if temporary_path: + try: + os.unlink(temporary_path) + except FileNotFoundError: + pass + + return True diff --git a/devenv/lib/limactl.py b/devenv/lib/limactl.py index 1db55196..a44f6305 100644 --- a/devenv/lib/limactl.py +++ b/devenv/lib/limactl.py @@ -2,97 +2,234 @@ import os import shutil +import stat import tempfile from devenv.constants import SYSTEM_MACHINE -from devenv.constants import home from devenv.constants import root from devenv.lib import archive from devenv.lib import proc +_DATA_DIRS = ("share/lima", "libexec/lima") +_GUEST_AGENT = { + "darwin_arm64": "lima-guestagent.Linux-aarch64.gz", + "darwin_x86_64": "lima-guestagent.Linux-x86_64.gz", +} +_DRIVER = {"darwin_arm64": "lima-driver-krunkit"} -def uninstall(binroot: str) -> None: - for d in (f"{home}/.lima",): - shutil.rmtree(d, ignore_errors=True) - for fp in ( +def _remove_managed_path(path: str) -> None: + try: + mode = os.lstat(path).st_mode + except FileNotFoundError: + return + + if stat.S_ISDIR(mode): + shutil.rmtree(path) + else: + os.remove(path) + + +def _managed_paths(binroot: str) -> tuple[str, ...]: + prefix = os.path.dirname(binroot) + return ( + f"{prefix}/share/lima", + f"{prefix}/libexec/lima", + f"{binroot}/templates", f"{binroot}/lima", f"{binroot}/limactl", f"{binroot}/lima-guestagent.Linux-aarch64", f"{binroot}/lima-guestagent.Linux-x86_64", - f"{binroot}/templates/default.yaml", - ): - try: - os.remove(fp) - except FileNotFoundError: - # it's better to do this than to guard with - # os.path.exists(fp) because if it's an invalid or circular - # symlink the result'll be False! - pass + ) -def _install(url: str, sha256: str, into: str) -> None: +def uninstall(binroot: str) -> None: + for path in _managed_paths(binroot): + _remove_managed_path(path) + + +def _install( + url: str, sha256: str, into: str, *, expected_version: str | None = None +) -> None: + prefix = os.path.dirname(into) os.makedirs(into, exist_ok=True) with tempfile.TemporaryDirectory(dir=into) as tmpd: archive_file = archive.download(url, sha256, dest=f"{tmpd}/download") - - # the archive from homebrew has a lima/version prefix - archive.unpack_strip_n(archive_file, tmpd, n=2) - - # the archive was atomically placed into tmpd so - # these are on the same fs and can be atomically moved too - os.replace(f"{tmpd}/bin/lima", f"{into}/lima") - os.replace(f"{tmpd}/bin/limactl", f"{into}/limactl") - os.replace( - f"{tmpd}/share/lima/lima-guestagent.Linux-aarch64", - f"{into}/lima-guestagent.Linux-aarch64", - ) - os.replace( - f"{tmpd}/share/lima/lima-guestagent.Linux-x86_64", - f"{into}/lima-guestagent.Linux-x86_64", - ) - os.makedirs(f"{into}/templates", exist_ok=True) - os.replace( - f"{tmpd}/share/lima/templates/default.yaml", - f"{into}/templates/default.yaml", - ) + archive.unpack(archive_file, tmpd) + missing = _missing_required_payload(tmpd) + if missing: + raise RuntimeError( + f"Lima release is incomplete: {', '.join(missing)}" + ) + + if expected_version is not None: + staged_version = _staged_version(tmpd) + if staged_version != expected_version: + raise RuntimeError( + f"Lima release reported {staged_version}, expected " + f"{expected_version}" + ) + + backup = tempfile.mkdtemp(prefix=".lima-install-", dir=prefix) + backed_up: list[tuple[str, str]] = [] + installed: list[str] = [] + try: + for target in _managed_paths(into): + if not os.path.lexists(target): + continue + backup_target = os.path.join( + backup, os.path.relpath(target, prefix) + ) + os.makedirs(os.path.dirname(backup_target), exist_ok=True) + os.replace(target, backup_target) + backed_up.append((backup_target, target)) + + for directory in _DATA_DIRS: + destination = f"{prefix}/{directory}" + os.makedirs(os.path.dirname(destination), exist_ok=True) + os.replace(f"{tmpd}/{directory}", destination) + installed.append(destination) + + for name in ("lima", "limactl"): + destination = f"{into}/{name}" + os.replace(f"{tmpd}/bin/{name}", destination) + installed.append(destination) + + missing = _missing_required_payload(prefix) + if missing: + raise RuntimeError( + f"Installed Lima release is incomplete: {', '.join(missing)}" + ) + if expected_version is not None: + installed_version = _installed_version(into) + if installed_version != expected_version: + raise RuntimeError( + f"Installed Lima reported {installed_version}, expected " + f"{expected_version}" + ) + except BaseException as error: + rollback_errors = [] + for target in reversed(installed): + try: + _remove_managed_path(target) + except OSError as rollback_error: + rollback_errors.append(str(rollback_error)) + for backup_target, target in reversed(backed_up): + try: + os.makedirs(os.path.dirname(target), exist_ok=True) + os.replace(backup_target, target) + except OSError as rollback_error: + rollback_errors.append(str(rollback_error)) + if rollback_errors: + raise RuntimeError( + f"Lima installation failed: {error}. Could not restore " + f"the previous installation: {'; '.join(rollback_errors)}. " + f"Previous files remain in {backup}." + ) from error + shutil.rmtree(backup) + raise + shutil.rmtree(backup) + + +def _staged_version(prefix: str) -> str | None: + versions = [] + for name in ("lima", "limactl"): + try: + env = ( + {"LIMACTL": f"{prefix}/bin/limactl"} if name == "lima" else None + ) + stdout = proc.run( + (f"{prefix}/bin/{name}", "--version"), stdout=True, env=env + ) + except RuntimeError: + return None + words = stdout.strip().split() + if not words: + return None + versions.append(words[-1]) + return versions[0] if versions[0] == versions[1] else "/".join(versions) + + +def _installed_version(binroot: str) -> str | None: + if any( + shutil.which(name, path=binroot) != f"{binroot}/{name}" + for name in ("lima", "limactl") + ): + return None + versions = [] + for name in ("lima", "limactl"): + try: + env = {"LIMACTL": f"{binroot}/limactl"} if name == "lima" else None + stdout = proc.run( + (f"{binroot}/{name}", "--version"), stdout=True, env=env + ) + except RuntimeError: + return None + words = stdout.strip().split() + if not words: + return None + versions.append(words[-1]) + return versions[0] if versions[0] == versions[1] else "/".join(versions) + + +def _missing_required_payload(prefix: str) -> tuple[str, ...]: + required_files = [ + f"{prefix}/bin/lima", + f"{prefix}/bin/limactl", + f"{prefix}/share/lima/{_GUEST_AGENT[SYSTEM_MACHINE]}", + f"{prefix}/share/lima/templates/default.yaml", + ] + if driver := _DRIVER.get(SYSTEM_MACHINE): + required_files.append(f"{prefix}/libexec/lima/{driver}") + + missing = [ + path + for path in required_files + if not os.path.isfile(path) or os.path.getsize(path) == 0 + ] + executable_files = [f"{prefix}/bin/lima", f"{prefix}/bin/limactl"] + if driver: + executable_files.append(f"{prefix}/libexec/lima/{driver}") + for path in executable_files: + if os.path.isfile(path) and not os.access(path, os.X_OK): + missing.append(path) + return tuple(missing) def install_global() -> None: - version = "1.0.3" + version = "2.2.1" cfg = { - # upstream github releases aren't built for macOS 14, so for now - # we use homebrew binaries from https://formulae.brew.sh/api/formula/lima.json - # unfortunately there's no way to view previous versions of a formula - # so if we ever need to upgrade to a specific non-latest version we'll - # have to build it ourselves - # sonoma - "darwin_x86_64": "https://ghcr.io/v2/homebrew/core/lima/blobs/sha256:f675abee28d0f10f335f7a04dc6ad3de12434c83c2f0f32c913061204c137a94", - "darwin_x86_64_sha256": "f675abee28d0f10f335f7a04dc6ad3de12434c83c2f0f32c913061204c137a94", - # arm64_sonoma - "darwin_arm64": "https://ghcr.io/v2/homebrew/core/lima/blobs/sha256:8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a", - "darwin_arm64_sha256": "8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a", + "darwin_x86_64": f"https://github.com/lima-vm/lima/releases/download/v{version}/lima-{version}-Darwin-x86_64.tar.gz", + "darwin_x86_64_sha256": "6b776e4bc41f358e6ee01fefcc78e2eb26f11635a0fcff632234af77a566c6c4", + "darwin_arm64": f"https://github.com/lima-vm/lima/releases/download/v{version}/lima-{version}-Darwin-arm64.tar.gz", + "darwin_arm64_sha256": "9e9eacce88f37e185c346bad73aa6136f738d8cdf8c3bb23cd42b071824bc66e", } binroot = f"{root}/bin" + installed_version = _installed_version(binroot) + missing = _missing_required_payload(root) + if installed_version == version and not missing: + return + + if installed_version is not None: + if installed_version != version: + print(f"installed limactl {installed_version} is unexpected!") + if missing: + print( + f"limactl {version} is missing {', '.join(missing)}; reinstalling" + ) - if ( - shutil.which("lima", path=binroot) == f"{binroot}/lima" - and shutil.which("limactl", path=binroot) == f"{binroot}/limactl" - ): - stdout = proc.run((f"{binroot}/limactl", "--version"), stdout=True) - installed_version = stdout.strip().split()[-1] - if version == installed_version: - return - print(f"installed limactl {installed_version} is unexpected!") - - uninstall(binroot) - _install(cfg[SYSTEM_MACHINE], cfg[f"{SYSTEM_MACHINE}_sha256"], binroot) + _install( + cfg[SYSTEM_MACHINE], + cfg[f"{SYSTEM_MACHINE}_sha256"], + binroot, + expected_version=version, + ) - stdout = proc.run((f"{binroot}/limactl", "--version"), stdout=True) - if f"limactl version {version}" not in stdout: + installed_version = _installed_version(binroot) + if installed_version != version: raise SystemExit( - f"Failed to install limactl {version}! Found: {stdout}" + f"Failed to install Lima {version}! Found: {installed_version}" ) diff --git a/devenv/lib/proc.py b/devenv/lib/proc.py index 1bfcb488..2bf3c65f 100644 --- a/devenv/lib/proc.py +++ b/devenv/lib/proc.py @@ -2,6 +2,7 @@ import shlex import subprocess +import sys from pathlib import Path from typing import Literal from typing import overload @@ -79,6 +80,9 @@ def run( if constants.DEBUG: xtrace(cmd) + # keep what we've printed ahead of the command's output + # when stdout isn't a tty and so is block buffered + sys.stdout.flush() try: proc = subprocess.run( cmd, check=True, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env diff --git a/tests/checks/test_colimaSve.py b/tests/checks/test_colimaSve.py new file mode 100644 index 00000000..ab814019 --- /dev/null +++ b/tests/checks/test_colimaSve.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from unittest import mock + +from devenv.checks import colimaSve +from devenv.lib import colima + + +def test_check_ok() -> None: + with mock.patch.object(colima, "vm_misreports_sve", return_value=False): + assert colimaSve.check() == (True, "") + + +def test_check_misreported() -> None: + with mock.patch.object(colima, "vm_misreports_sve", return_value=True): + assert colimaSve.check() == ( + False, + "colima's VM kernel reports SVE2, but this CPU has no SVE. " + "Programs that use it crash with 'Illegal instruction'.", + ) + + +def test_check_colima_not_running() -> None: + # another check reports that, and its fix starts colima, + # which fixes this too + with mock.patch.object( + colima, + "vm_misreports_sve", + side_effect=RuntimeError("Command `colima ssh` failed! (code 1)"), + ): + assert colimaSve.check() == (True, "") + + +def test_fix() -> None: + with ( + mock.patch.object( + colima, "start", return_value=colima.ColimaStatus.UP + ) as start, + mock.patch.object(colima, "vm_misreports_sve", return_value=False), + ): + assert colimaSve.fix() == (True, "") + start.assert_called_once_with(restart=True) + + +def test_fix_didnt_take() -> None: + # e.g. apt couldn't reach the mirror, which start only warns about + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "vm_misreports_sve", return_value=True), + ): + assert colimaSve.fix() == ( + False, + "colima's VM still misreports SVE, see above", + ) + + +def test_fix_unhealthy() -> None: + with mock.patch.object( + colima, "start", return_value=colima.ColimaStatus.UNHEALTHY + ): + assert colimaSve.fix() == (False, "colima is unhealthy") diff --git a/tests/checks/test_limaDns.py b/tests/checks/test_limaDns.py new file mode 100644 index 00000000..76bdbe0d --- /dev/null +++ b/tests/checks/test_limaDns.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +from unittest import mock + +import pytest + +from devenv.checks import limaDns +from devenv.lib import colima +from devenv.lib import proc + + +@pytest.mark.parametrize( + ("target", "resolv_conf", "service"), + ( + ( + "/run/systemd/resolve/stub-resolv.conf", + "nameserver 127.0.0.53\n", + "systemd-resolved.service", + ), + ( + "/etc/resolv.conf", + "# Generated by Colima\nnameserver 192.168.5.1\n", + "dnsmasq.service", + ), + ( + "/etc/resolv.conf", + "nameserver 127.0.0.53\n", + "systemd-resolved.service", + ), + ), +) +def test_resolver_service(target: str, resolv_conf: str, service: str) -> None: + assert limaDns._resolver_service(target, resolv_conf) == service + + +def test_unknown_resolver() -> None: + assert ( + limaDns._resolver_service( + "/etc/resolv.conf", "nameserver 192.168.5.1\n" + ) + is None + ) + + +@pytest.mark.parametrize( + ("target", "resolv_conf", "service"), + ( + ( + "/run/systemd/resolve/stub-resolv.conf", + "nameserver 127.0.0.53\n", + "systemd-resolved.service", + ), + ( + "/etc/resolv.conf", + "# Generated by Colima\nnameserver 192.168.5.1\n", + "dnsmasq.service", + ), + ), +) +def test_fix_restarts_selected_resolver( + target: str, resolv_conf: str, service: str +) -> None: + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object( + proc, "run", side_effect=(f"{target}\n{resolv_conf}", "", "") + ) as run, + ): + assert limaDns.fix() == (True, "") + + assert run.call_args_list[1].args[0][-1] == service + assert ( + 'socket.getaddrinfo("ghcr.io", None)' + in run.call_args_list[2].args[0][-1] + ) + + +def test_fix_reports_unknown_resolver() -> None: + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object( + proc, + "run", + return_value="/etc/resolv.conf\nnameserver 192.168.5.1\n", + ), + ): + assert limaDns.fix() == ( + False, + "Unable to identify Colima's DNS resolver from /etc/resolv.conf. " + "Inspect the VM resolver configuration and restart the selected " + "resolver manually.", + ) + + +def test_fix_restarts_colima_when_resolver_restart_fails() -> None: + commands: list[tuple[str, ...]] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + commands.append(command) + if len(commands) == 1: + return ( + "/etc/resolv.conf\n" + "# Generated by Colima\nnameserver 192.168.5.1\n" + ) + if command[-1] == "dnsmasq.service": + raise RuntimeError("dnsmasq restart failed") + if "socket.getaddrinfo" in command[-1]: + return "" + return "" + + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(proc, "run", side_effect=run), + ): + assert limaDns.fix() == (True, "") + + assert commands[-2][-4:] == ("-m", "devenv", "colima", "restart") + + +def test_fix_reports_colima_restart_failure() -> None: + commands = 0 + + def run(command: tuple[str, ...], **kwargs: object) -> str: + nonlocal commands + commands += 1 + if commands == 1: + return ( + "/etc/resolv.conf\n" + "# Generated by Colima\nnameserver 192.168.5.1\n" + ) + if command[0] == "colima": + raise RuntimeError("dnsmasq restart failed") + raise RuntimeError("colima restart failed") + + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(proc, "run", side_effect=run), + ): + assert limaDns.fix() == (False, "colima restart failed") + + +def test_check_runs_real_dns_resolution_probe() -> None: + commands: list[tuple[str, ...]] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + commands.append(command) + return "" + + with ( + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(proc, "run", side_effect=run), + ): + assert limaDns.check() == (True, "") + + assert 'socket.getaddrinfo("ghcr.io", None)' in commands[0][-1] + + +def test_check_reports_resolution_failure() -> None: + with ( + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object( + proc, "run", side_effect=RuntimeError("failed to resolve ghcr.io") + ), + ): + assert limaDns.check() == (False, "failed to resolve ghcr.io") + + +def test_dnsmasq_restart_repairs_resolution() -> None: + dnsmasq_running = False + + def run(command: tuple[str, ...], **kwargs: object) -> str: + nonlocal dnsmasq_running + script = command[-1] + if 'Path("/etc/resolv.conf")' in script: + return ( + "/etc/resolv.conf\n" + "# Generated by Colima\n\n" + "nameserver 192.168.5.1\n" + ) + if 'socket.getaddrinfo("ghcr.io", None)' in script: + if dnsmasq_running: + return "" + raise RuntimeError("failed to resolve ghcr.io") + if command[-1] == "dnsmasq.service": + dnsmasq_running = True + return "" + raise AssertionError(command) + + with ( + mock.patch.object(colima, "start", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(proc, "run", side_effect=run), + ): + assert limaDns.check() == (False, "failed to resolve ghcr.io") + assert limaDns.fix() == (True, "") + assert limaDns.check() == (True, "") + + assert dnsmasq_running diff --git a/tests/lib/test_colima.py b/tests/lib/test_colima.py index 0938b16b..e5c0daf1 100644 --- a/tests/lib/test_colima.py +++ b/tests/lib/test_colima.py @@ -1,17 +1,429 @@ from __future__ import annotations -from unittest.mock import patch +from collections.abc import Generator +from pathlib import Path +from unittest import mock +import pytest + +from devenv import constants from devenv.lib import colima -from devenv.lib.colima import ColimaStatus + +# /proc/cpuinfo in colima 0.8.1's VM (Linux 6.8.0-50-generic) on an M4 Pro +M4_BUGGY_FEATURES = ( + "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid " + "asimdrdm jscvt fcma lrcpc dcpop sha3 asimddp sha512 asimdfhm dit uscat " + "ilrcpc flagm sb paca pacg dcpodp sve2 flagm2 frint svei8mm svebf16 i8mm " + "bf16 bti afp sme smei16i64 smef64f64 smei8i32 smef16f32 smeb16f32 " + "smef32f32 sme2 smei16i32 smebi32i32" +) +# ... and after upgrading it to Linux 6.8.0-147-generic +M4_FIXED_FEATURES = ( + "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid " + "asimdrdm jscvt fcma lrcpc dcpop sha3 asimddp sha512 asimdfhm dit uscat " + "ilrcpc flagm sb paca pacg dcpodp flagm2 frint i8mm bf16 bti afp" +) + +BUGGY_KERNEL = "6.8.0-50-generic" +FIXED_KERNEL = "6.8.0-147-generic" + +SYSCTL_SME = ("/usr/sbin/sysctl", "-n", "hw.optional.arm.FEAT_SME") +APT_UPDATE = ("colima", "ssh", "--", *colima._APT_GET, "update") +APT_INSTALL = ( + "colima", + "ssh", + "--", + *colima._APT_GET, + "install", + "-y", + "linux-image-virtual", +) + + +class FakeVM: + """Answers the commands devenv runs on the host and in colima's VM.""" + + def __init__( + self, + *, + running: str = BUGGY_KERNEL, + available: str = FIXED_KERNEL, + buggy: frozenset[str] = frozenset((BUGGY_KERNEL,)), + host_sme: bool = True, + apt_works: bool = True, + boots_newest: bool = True, + start_results: tuple[colima.ColimaStatus, ...] = (), + ) -> None: + self.running = running + self.installed = {running} + self.available = available + self.buggy = buggy + self.host_sme = host_sme + self.apt_works = apt_works + self.boots_newest = boots_newest + self.cmds: list[tuple[str, ...]] = [] + self.events: list[tuple[str, object]] = [] + self.restarts = 0 + self.recoveries = 0 + self.start_modes: list[colima._StartMode] = [] + self.start_existing_instances: list[bool] = [] + self.start_results = list(start_results) + + def run( + self, + cmd: tuple[str, ...], + *, + pathprepend: str = "", + stdout: bool = False, + ) -> str | None: + self.cmds.append(cmd) + self.events.append(("command", cmd)) + if cmd == SYSCTL_SME: + if not self.host_sme: + raise RuntimeError("sysctl: unknown oid") + return "1" + if cmd == ("colima", "stop"): + return None + if cmd in (APT_UPDATE, APT_INSTALL): + if not self.apt_works: + raise RuntimeError("Command `apt-get` failed! (code 100)") + if cmd == APT_INSTALL: + self.installed.add(self.available) + return None + + assert cmd[:3] == ("colima", "ssh", "--"), cmd + vm_cmd = cmd[3:] + if vm_cmd == ("grep", "-m1", "^Features", "/proc/cpuinfo"): + if self.running in self.buggy: + return f"Features\t: {M4_BUGGY_FEATURES}" + return f"Features\t: {M4_FIXED_FEATURES}" + if vm_cmd == ("uname", "-r"): + return self.running + if vm_cmd == ("linux-version", "list"): + return "\n".join(sorted(self.installed)) + if vm_cmd == ("sync",): + return None + raise AssertionError(f"unexpected command: {cmd}") + + def start( + self, + mode: colima._StartMode, + status: colima.ColimaStatus | None = None, + *, + existing_instance: bool = False, + ) -> colima.ColimaStatus: + self.start_modes.append(mode) + self.start_existing_instances.append(existing_instance) + self.events.append(("start", mode)) + if mode == colima._StartMode.REQUESTED_RESTART: + self.restarts += 1 + elif mode == colima._StartMode.HEALTH_RECOVERY: + self.recoveries += 1 + if mode in { + colima._StartMode.REQUESTED_RESTART, + colima._StartMode.HEALTH_RECOVERY, + }: + if self.boots_newest: + self.running = max( + self.installed, key=colima._kernel_version_key + ) + if self.start_results: + return self.start_results.pop(0) + return colima.ColimaStatus.UP + + def ran(self, cmd: tuple[str, ...]) -> bool: + return cmd in self.cmds + + +@pytest.fixture +def m4_mac() -> Generator[None]: + with ( + mock.patch.object(constants, "DARWIN", True), + mock.patch.object(constants, "MACHINE", "arm64"), + ): + yield + + +def _start( + vm: FakeVM, + restart: bool = False, + initial_status: colima.ColimaStatus = colima.ColimaStatus.UP, +) -> colima.ColimaStatus: + with ( + mock.patch("devenv.lib.colima.proc.run", side_effect=vm.run), + mock.patch.object(colima, "_start", side_effect=vm.start), + mock.patch.object(colima, "check", return_value=initial_status), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + ): + return colima.start(restart) + + +@pytest.mark.parametrize( + ("features", "expected"), + ( + # M4 and newer with an unfixed kernel + (M4_BUGGY_FEATURES, True), + (M4_FIXED_FEATURES, False), + # e.g. AWS Graviton 3, which really has SVE + ("fp asimd sve sve2 svei8mm svebf16 i8mm bf16", False), + # M1-M3, no SVE or SME + ("fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics", False), + ), +) +def test_misreports_sve(features: str, expected: bool) -> None: + assert colima._misreports_sve(features.split()) is expected + + +def test_kernel_version_key_sorts_abi_numerically() -> None: + newest = max( + ("6.8.0-50-generic", "6.8.0-147-generic", "6.8.0-64-generic"), + key=colima._kernel_version_key, + ) + assert newest == "6.8.0-147-generic" + + +@pytest.mark.parametrize( + ("darwin", "machine"), + ((False, "x86_64"), (False, "aarch64"), (True, "x86_64")), +) +def test_vm_isnt_probed_on_linux_or_intel_macs( + darwin: bool, machine: str +) -> None: + vm = FakeVM() + with ( + mock.patch.object(constants, "DARWIN", darwin), + mock.patch.object(constants, "MACHINE", machine), + mock.patch("devenv.lib.colima.proc.run", side_effect=vm.run), + ): + assert colima.vm_misreports_sve() is False + assert vm.cmds == [] + + +@pytest.mark.usefixtures("m4_mac") +def test_vm_isnt_probed_on_macs_without_sme() -> None: + vm = FakeVM(host_sme=False) + with mock.patch("devenv.lib.colima.proc.run", side_effect=vm.run): + assert colima.vm_misreports_sve() is False + assert vm.cmds == [SYSCTL_SME] + + +@pytest.mark.usefixtures("m4_mac") +def test_start_leaves_a_fixed_vm_alone() -> None: + vm = FakeVM(running=FIXED_KERNEL) + + assert _start(vm) == colima.ColimaStatus.UP + + assert vm.restarts == 0 + assert not vm.ran(APT_INSTALL) + + +@pytest.mark.usefixtures("m4_mac") +def test_start_upgrades_the_kernel_and_restarts_once( + capsys: pytest.CaptureFixture[str], +) -> None: + vm = FakeVM() + + assert _start(vm) == colima.ColimaStatus.UP + + assert vm.ran(APT_UPDATE) + assert vm.ran(APT_INSTALL) + assert vm.events.index(("command", ("colima", "ssh", "--", "sync"))) < ( + vm.events.index(("start", colima._StartMode.REQUESTED_RESTART)) + ) + assert vm.restarts == 1 + assert vm.running == FIXED_KERNEL + assert f"colima's VM is now running Linux {FIXED_KERNEL}." in ( + capsys.readouterr().out + ) + + # and a second start has nothing left to do + vm.cmds.clear() + assert _start(vm) == colima.ColimaStatus.UP + assert vm.restarts == 1 + assert not vm.ran(APT_INSTALL) + + +@pytest.mark.usefixtures("m4_mac") +def test_start_restarts_only_once_if_the_new_kernel_doesnt_help( + capsys: pytest.CaptureFixture[str], +) -> None: + vm = FakeVM(buggy=frozenset((BUGGY_KERNEL, FIXED_KERNEL))) + + assert _start(vm) == colima.ColimaStatus.UP + + assert vm.restarts == 1 + assert vm.cmds.count(APT_INSTALL) == 1 + assert "still misreports SVE support" in capsys.readouterr().out + + # the newest kernel's running, but there's nothing newer to upgrade to + vm.cmds.clear() + assert _start(vm) == colima.ColimaStatus.UP + assert vm.restarts == 1 + assert f"no kernel newer than Linux {FIXED_KERNEL}" in ( + capsys.readouterr().out + ) + + +@pytest.mark.usefixtures("m4_mac") +def test_start_doesnt_keep_restarting_into_an_old_kernel( + capsys: pytest.CaptureFixture[str], +) -> None: + # e.g. grub pinned to an older kernel + vm = FakeVM(boots_newest=False) + + assert _start(vm) == colima.ColimaStatus.UP + assert vm.restarts == 1 + assert vm.running == BUGGY_KERNEL + assert "still misreports SVE support" in capsys.readouterr().out + + for _ in range(2): + vm.cmds.clear() + assert _start(vm) == colima.ColimaStatus.UP + assert vm.restarts == 1 + assert not vm.ran(APT_INSTALL) + assert "Run `devenv colima restart` to boot it" in ( + capsys.readouterr().out + ) + + +@pytest.mark.usefixtures("m4_mac") +def test_restart_installs_the_kernel_before_its_one_reboot() -> None: + vm = FakeVM() + + assert _start(vm, restart=True) == colima.ColimaStatus.UP + + assert vm.restarts == 1 + assert vm.ran(APT_INSTALL) + assert vm.running == FIXED_KERNEL + + +@pytest.mark.usefixtures("m4_mac") +def test_restart_boots_an_installed_kernel_after_an_interrupted_repair( + capsys: pytest.CaptureFixture[str], +) -> None: + # e.g. an earlier upgrade was interrupted before colima restarted + vm = FakeVM() + vm.installed.add(FIXED_KERNEL) + + assert _start(vm, restart=True) == colima.ColimaStatus.UP + + assert vm.running == FIXED_KERNEL + assert vm.restarts == 1 + assert not vm.ran(APT_INSTALL) + output = capsys.readouterr().out + assert f"colima's VM is now running Linux {FIXED_KERNEL}." in output + assert "Run `devenv colima restart`" not in output + + +@pytest.mark.usefixtures("m4_mac") +def test_start_carries_on_if_apt_fails( + capsys: pytest.CaptureFixture[str], +) -> None: + vm = FakeVM(apt_works=False) + + assert _start(vm) == colima.ColimaStatus.UP + + assert vm.restarts == 0 + assert "failed to upgrade colima's VM kernel" in capsys.readouterr().out + + +@pytest.mark.usefixtures("m4_mac") +def test_start_carries_on_if_the_vm_cant_be_probed( + capsys: pytest.CaptureFixture[str], +) -> None: + def run(cmd: tuple[str, ...], **kwargs: object) -> str: + if cmd == SYSCTL_SME: + return "1" + raise RuntimeError("Command `colima ssh` failed! (code 1)") + + with ( + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + mock.patch.object( + colima, "_start", return_value=colima.ColimaStatus.UP + ), + ): + assert colima.start() == colima.ColimaStatus.UP + + assert "failed to check colima's VM kernel" in capsys.readouterr().out + + +@pytest.mark.usefixtures("m4_mac") +def test_start_doesnt_probe_an_unhealthy_vm() -> None: + vm = FakeVM() + with ( + mock.patch("devenv.lib.colima.proc.run", side_effect=vm.run), + mock.patch.object( + colima, "check", return_value=colima.ColimaStatus.UNHEALTHY + ), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + mock.patch.object( + colima, "_start", return_value=colima.ColimaStatus.UNHEALTHY + ), + ): + assert colima.start() == colima.ColimaStatus.UNHEALTHY + + assert vm.cmds == [] + + +@pytest.mark.usefixtures("m4_mac") +def test_start_warns_if_colima_doesnt_come_back_after_the_upgrade( + capsys: pytest.CaptureFixture[str], +) -> None: + vm = FakeVM( + start_results=( + colima.ColimaStatus.UP, + colima.ColimaStatus.UNHEALTHY, + colima.ColimaStatus.UNHEALTHY, + ) + ) + with ( + mock.patch("devenv.lib.colima.proc.run", side_effect=vm.run), + mock.patch.object(colima, "_start", side_effect=vm.start), + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + ): + assert colima.start() == colima.ColimaStatus.UNHEALTHY + + assert "isn't healthy after restarting" in capsys.readouterr().out + assert vm.restarts == 1 + assert vm.recoveries == 1 def test_start_only_selects_colima_context_when_up() -> None: with ( - patch("devenv.lib.colima.check", return_value=ColimaStatus.UP), - patch("devenv.lib.colima.proc.run") as mock_run, + mock.patch( + "devenv.lib.colima.check", return_value=colima.ColimaStatus.UP + ), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + mock.patch("devenv.lib.colima.proc.run") as mock_run, + mock.patch( + "devenv.lib.colima._upgrade_kernel_if_sve_misreported", + return_value=False, + ), ): - assert colima.start() == ColimaStatus.UP + assert colima.start() == colima.ColimaStatus.UP # no restart, but a previously interrupted start may have left docker # on the "default" context @@ -20,19 +432,27 @@ def test_start_only_selects_colima_context_when_up() -> None: def test_start_switches_context_around_starting() -> None: with ( - patch( + mock.patch( "devenv.lib.colima.check", - side_effect=[ColimaStatus.DOWN, ColimaStatus.UP], + side_effect=[colima.ColimaStatus.DOWN, colima.ColimaStatus.UP], + ), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + mock.patch("devenv.lib.colima.rosetta.ensure"), + mock.patch("devenv.lib.colima.proc.run") as mock_run, + mock.patch( + "devenv.lib.colima._upgrade_kernel_if_sve_misreported", + return_value=False, ), - patch("devenv.lib.colima.rosetta.ensure"), - patch("devenv.lib.colima.proc.run") as mock_run, ): - assert colima.start() == ColimaStatus.UP + assert colima.start() == colima.ColimaStatus.UP cmds = [c.args[0][:3] for c in mock_run.call_args_list] assert cmds == [ ("docker", "context", "use"), - ("colima", "stop", "-f"), ("colima", "start", "--verbose"), ("docker", "context", "use"), ] @@ -42,15 +462,300 @@ def test_start_switches_context_around_starting() -> None: def test_restart_when_up() -> None: with ( - patch( + mock.patch( "devenv.lib.colima.check", - side_effect=[ColimaStatus.UP, ColimaStatus.UP], + side_effect=[colima.ColimaStatus.UP, colima.ColimaStatus.UP], + ), + mock.patch.object( + colima, + "_prepare_docker_store", + return_value=colima._StoragePolicy(existing_instance=False), + ), + mock.patch("devenv.lib.colima.rosetta.ensure"), + mock.patch("devenv.lib.colima.proc.run") as mock_run, + mock.patch( + "devenv.lib.colima._upgrade_kernel_if_sve_misreported", + return_value=False, ), - patch("devenv.lib.colima.rosetta.ensure"), - patch("devenv.lib.colima.proc.run") as mock_run, ): - assert colima.restart() == ColimaStatus.UP + assert colima.restart() == colima.ColimaStatus.UP cmds = [c.args[0][:2] for c in mock_run.call_args_list] assert ("colima", "stop") in cmds + assert ("colima", "stop", "-f") not in [ + c.args[0][:3] for c in mock_run.call_args_list + ] assert ("colima", "start") in cmds + + +def test_unhealthy_start_gets_one_bounded_recovery() -> None: + from devenv import colima as colima_cmd + from devenv.lib.context import Context + + context: Context = {"config_path": "", "code_root": "", "repo": None} + with mock.patch( + "devenv.colima.colima.start", return_value=colima.ColimaStatus.UNHEALTHY + ) as start: + assert colima_cmd.main(context, ["start"]) == 1 + + start.assert_called_once_with() + + +@pytest.mark.usefixtures("m4_mac") +def test_post_start_health_recovery_still_repairs_the_kernel() -> None: + vm = FakeVM( + start_results=( + colima.ColimaStatus.UNHEALTHY, + colima.ColimaStatus.UP, + colima.ColimaStatus.UP, + ) + ) + + assert ( + _start(vm, initial_status=colima.ColimaStatus.DOWN) + == colima.ColimaStatus.UP + ) + + assert vm.start_modes == [ + colima._StartMode.ENSURE_RUNNING, + colima._StartMode.HEALTH_RECOVERY, + colima._StartMode.REQUESTED_RESTART, + ] + assert vm.start_existing_instances == [False, True, True] + assert vm.recoveries == 1 + assert vm.restarts == 1 + assert vm.running == FIXED_KERNEL + + +@pytest.mark.usefixtures("m4_mac") +def test_kernel_restart_recovery_validates_the_booted_kernel( + capsys: pytest.CaptureFixture[str], +) -> None: + vm = FakeVM( + start_results=( + colima.ColimaStatus.UP, + colima.ColimaStatus.UNHEALTHY, + colima.ColimaStatus.UP, + ) + ) + + assert _start(vm) == colima.ColimaStatus.UP + + assert vm.start_modes == [ + colima._StartMode.ENSURE_RUNNING, + colima._StartMode.REQUESTED_RESTART, + colima._StartMode.HEALTH_RECOVERY, + ] + assert vm.start_existing_instances == [False, True, True] + assert vm.restarts == 1 + assert vm.recoveries == 1 + assert vm.running == FIXED_KERNEL + assert vm.cmds.count(APT_INSTALL) == 1 + assert vm.ran( + ("colima", "ssh", "--", "grep", "-m1", "^Features", "/proc/cpuinfo") + ) + assert f"colima's VM is now running Linux {FIXED_KERNEL}." in ( + capsys.readouterr().out + ) + + +def test_start_from_down_does_not_force_stop() -> None: + with ( + mock.patch.object(colima, "rosetta") as rosetta, + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "proc") as proc, + mock.patch("devenv.lib.colima.platform.machine", return_value="arm64"), + mock.patch("devenv.lib.colima.os.cpu_count", return_value=8), + mock.patch( + "devenv.lib.colima.os.sysconf", side_effect=(4096, 8_388_608) + ), + ): + assert ( + colima._start( + colima._StartMode.ENSURE_RUNNING, + colima.ColimaStatus.DOWN, + existing_instance=False, + ) + == colima.ColimaStatus.UP + ) + + rosetta.ensure.assert_called_once_with() + commands = [call.args[0] for call in proc.run.call_args_list] + assert ("colima", "stop", "-f") not in commands + start = next( + command for command in commands if command[:2] == ("colima", "start") + ) + assert "--cpus" in start + assert "--cpu" not in start + assert "--disk" in start and "512" in start + assert "--save-config" in start + assert "--mount-inotify" not in start + assert "--vm-type=vz" in start + assert "--vz-rosetta" in start + assert "--mount-type=virtiofs" in start + + +def test_existing_start_preserves_saved_vm_settings() -> None: + with ( + mock.patch.object(colima, "rosetta") as rosetta, + mock.patch.object(colima, "check", return_value=colima.ColimaStatus.UP), + mock.patch.object(colima, "proc") as proc, + mock.patch("devenv.lib.colima.os.cpu_count", return_value=8), + mock.patch( + "devenv.lib.colima.os.sysconf", side_effect=(4096, 8_388_608) + ), + ): + assert ( + colima._start( + colima._StartMode.REQUESTED_RESTART, + colima.ColimaStatus.UP, + existing_instance=True, + ) + == colima.ColimaStatus.UP + ) + + rosetta.ensure.assert_called_once_with() + commands = [call.args[0] for call in proc.run.call_args_list] + assert ("colima", "stop") in commands + assert ("colima", "stop", "-f") not in commands + start = next( + command for command in commands if command[:2] == ("colima", "start") + ) + assert start == ("colima", "start", "--verbose") + assert "--mount-inotify" not in start + + +def test_colima_upgrade_and_uninstall_preserve_standalone_lima_data( + tmp_path: Path, +) -> None: + home = tmp_path / "home" + sentinel = home / ".lima/default/lima.yaml" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"independent lima vm\x00state") + original = sentinel.read_bytes() + colima_disk = home / ".colima/_lima/colima/diffdisk" + colima_disk.parent.mkdir(parents=True) + colima_disk.write_bytes(b"persistent Colima data") + original_colima_disk = colima_disk.read_bytes() + binroot = tmp_path / "sentry-devenv/bin" + binroot.mkdir(parents=True) + unrelated_binary = binroot / "docker" + unrelated_binary.write_bytes(b"unrelated executable") + old_shim = binroot / "colima" + old_shim.write_text("#!/bin/sh\nexit 0\n") + old_shim.chmod(0o755) + old_binary = binroot / "colima-bin" + old_binary.write_bytes(b"old Colima executable") + downloads: list[tuple[str, str, str]] = [] + versions = iter(("colima version v0.8.1", "colima version v0.10.3")) + + def which(command: str, path: str | None = None) -> str | None: + if command == "colima" and path == str(binroot): + return str(binroot / "colima") + return None + + def install(url: str, sha256: str, into: str) -> None: + downloads.append((url, sha256, into)) + binary = Path(into) / "colima-bin" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o775) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(tmp_path / "sentry-devenv")), + mock.patch.object(colima, "SYSTEM_MACHINE", "darwin_arm64"), + mock.patch("devenv.lib.colima.shutil.which", side_effect=which), + mock.patch( + "devenv.lib.colima.proc.run", + side_effect=lambda *a, **k: next(versions), + ), + mock.patch("devenv.lib.colima._install", side_effect=install), + ): + colima.install_global() + assert sentinel.read_bytes() == original + assert colima_disk.read_bytes() == original_colima_disk + assert unrelated_binary.read_bytes() == b"unrelated executable" + colima.uninstall(str(binroot)) + + assert downloads == [ + ( + "https://github.com/abiosoft/colima/releases/download/v0.10.3/colima-Darwin-arm64", + "980ad8bf61a4ca370243f4cb41401a61276dcd2c2502bee7b9b86f9250169f34", + str(binroot), + ) + ] + assert sorted(path.name for path in binroot.iterdir()) == ["docker"] + assert unrelated_binary.read_bytes() == b"unrelated executable" + assert sentinel.read_bytes() == original + assert colima_disk.read_bytes() == original_colima_disk + + +def test_interrupted_colima_upgrade_preserves_installed_binary_and_lima_data( + tmp_path: Path, +) -> None: + home = tmp_path / "home" + sentinel = home / ".lima/default/lima.yaml" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"independent lima vm\x00state") + original = sentinel.read_bytes() + binroot = tmp_path / "sentry-devenv/bin" + binroot.mkdir(parents=True) + shim = binroot / "colima" + shim.write_text("#!/bin/sh\nexec colima-bin\n") + shim.chmod(0o755) + binary = binroot / "colima-bin" + binary.write_bytes(b"Colima 0.8.1") + original_binary = binary.read_bytes() + + with ( + mock.patch.object(colima, "root", str(tmp_path / "sentry-devenv")), + mock.patch.object(colima, "SYSTEM_MACHINE", "darwin_arm64"), + mock.patch("devenv.lib.colima.shutil.which", return_value=str(shim)), + mock.patch( + "devenv.lib.colima.proc.run", return_value="colima version v0.8.1" + ), + mock.patch( + "devenv.lib.colima._install", + side_effect=RuntimeError("download interrupted"), + ), + pytest.raises(RuntimeError, match="download interrupted"), + ): + colima.install_global() + + assert binary.read_bytes() == original_binary + assert shim.is_file() + assert sentinel.read_bytes() == original + + +def test_interrupted_colima_shim_is_reinstalled(tmp_path: Path) -> None: + binroot = tmp_path / "sentry-devenv/bin" + binroot.mkdir(parents=True) + shim = binroot / "colima" + shim.write_text("#!/bin/sh\nexec colima-bin\n") + shim.chmod(0o755) + versions = iter( + (RuntimeError("colima-bin is missing"), "colima version v0.10.3") + ) + + def installed_version(command: tuple[str, ...], **kwargs: object) -> str: + result = next(versions) + if isinstance(result, RuntimeError): + raise result + return result + + def install(url: str, sha256: str, into: str) -> None: + binary = Path(into) / "colima-bin" + binary.write_bytes(b"reinstalled Colima") + binary.chmod(0o775) + + with ( + mock.patch.object(colima, "root", str(tmp_path / "sentry-devenv")), + mock.patch.object(colima, "SYSTEM_MACHINE", "darwin_arm64"), + mock.patch("devenv.lib.colima.shutil.which", return_value=str(shim)), + mock.patch("devenv.lib.colima.proc.run", side_effect=installed_version), + mock.patch("devenv.lib.colima._install", side_effect=install), + ): + colima.install_global() + + assert (binroot / "colima-bin").read_bytes() == b"reinstalled Colima" + assert f"exec {binroot}/colima-bin" in shim.read_text() diff --git a/tests/lib/test_colima_config.py b/tests/lib/test_colima_config.py new file mode 100644 index 00000000..8c35114e --- /dev/null +++ b/tests/lib/test_colima_config.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import os +from pathlib import Path +from unittest import mock + +import pytest + +from devenv.lib import colima_config + + +def _write_config(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +@pytest.mark.parametrize("snapshotter", (True, False)) +def test_explicit_snapshotter_choice_is_preserved( + tmp_path: Path, snapshotter: bool +) -> None: + config_path = tmp_path / "colima.yaml" + value = str(snapshotter).lower() + _write_config( + config_path, + f"docker:\n features:\n buildkit: false\n " + f"containerd-snapshotter: {value}\n", + ) + original = config_path.read_bytes() + + parsed = colima_config.read(str(config_path)) + + assert parsed.snapshotter is snapshotter + assert ( + colima_config.set_snapshotter(str(config_path), not snapshotter) + is False + ) + assert config_path.read_bytes() == original + + +def test_adds_legacy_setting_without_rewriting_custom_docker_fields( + tmp_path: Path, +) -> None: + config_path = tmp_path / "colima.yaml" + original = ( + "disk: 512\n" + "runtime: docker\n" + "mountInotify: true\n" + "docker:\n" + " insecure-registries:\n" + " - registry.example.com\n" + " features:\n" + " buildkit: false\n" + "network:\n" + " dns: [8.8.8.8]\n" + ) + _write_config(config_path, original) + + assert colima_config.set_snapshotter(str(config_path), False) is True + + updated = config_path.read_text() + assert "insecure-registries:\n - registry.example.com" in updated + assert "buildkit: false" in updated + assert "containerd-snapshotter: false" in updated + assert "dns: [8.8.8.8]" in updated + assert "mountInotify: true" in updated + assert colima_config.set_snapshotter(str(config_path), False) is False + + +@pytest.mark.parametrize( + ("docker", "expected"), + ( + ("docker: {}\n", "docker:\n features:\n"), + ("docker:\n log-level: warn\n", " log-level: warn\n features:\n"), + ("runtime: docker\n", "docker:\n features:\n"), + ), +) +def test_adds_feature_mapping_to_common_generated_shapes( + tmp_path: Path, docker: str, expected: str +) -> None: + config_path = tmp_path / "colima.yaml" + _write_config(config_path, docker) + + colima_config.set_snapshotter(str(config_path), False) + + updated = config_path.read_text() + assert expected in updated + assert "buildkit: true" in updated + assert "containerd-snapshotter: false" in updated + + +@pytest.mark.parametrize("text", ("docker:", "docker:\n features:")) +def test_adds_feature_mapping_when_file_has_no_final_newline( + tmp_path: Path, text: str +) -> None: + config_path = tmp_path / "colima.yaml" + _write_config(config_path, text) + + colima_config.set_snapshotter(str(config_path), False) + + updated = config_path.read_text() + assert "docker:\n" in updated + assert " features:\n" in updated + assert " containerd-snapshotter: false\n" in updated + + +@pytest.mark.parametrize( + "text", + ( + "docker:\n features: {buildkit: true}\n", + "docker:\n features:\n containerd-snapshotter: maybe\n", + "docker:\n features:\n buildkit: true\n buildkit: false\n", + 'docker:\n features:\n "containerd-snapshotter": true\n', + "docker:\n features:\n containerd-snapshotter: false\n", + "docker : {}\n", + "\ufeffdocker: {}\n", + "{docker: {}}\n", + "---\ndocker: {}\n", + "docker: null\n", + "docker:\n features:\n\tbuildkit: true\n", + ), +) +def test_unsupported_config_is_not_rewritten(tmp_path: Path, text: str) -> None: + config_path = tmp_path / "colima.yaml" + _write_config(config_path, text) + original = config_path.read_bytes() + + with pytest.raises(colima_config.StorageMigrationBlocked): + colima_config.set_snapshotter(str(config_path), False) + + assert config_path.read_bytes() == original + + +def test_config_symlink_is_not_replaced(tmp_path: Path) -> None: + target = tmp_path / "source.yaml" + link = tmp_path / "colima.yaml" + _write_config(target, "docker: {}\n") + link.symlink_to(target) + + with pytest.raises(colima_config.StorageMigrationBlocked): + colima_config.set_snapshotter(str(link), False) + + assert link.is_symlink() + assert target.read_text() == "docker: {}\n" + + +def test_non_regular_config_is_not_read(tmp_path: Path) -> None: + config_path = tmp_path / "colima.yaml" + os.mkfifo(config_path) + + with pytest.raises( + colima_config.StorageMigrationBlocked, match="not a regular file" + ): + colima_config.set_snapshotter(str(config_path), False) + + +def test_atomic_write_preserves_mode_and_original_on_replace_failure( + tmp_path: Path, +) -> None: + config_path = tmp_path / "colima.yaml" + original = "docker: {}\n" + _write_config(config_path, original) + config_path.chmod(0o640) + + with mock.patch( + "devenv.lib.colima_config.os.replace", + side_effect=OSError("rename failed"), + ): + with pytest.raises(colima_config.StorageMigrationBlocked): + colima_config.set_snapshotter(str(config_path), False) + + assert config_path.read_text() == original + assert os.stat(config_path).st_mode & 0o777 == 0o640 + assert list(tmp_path.iterdir()) == [config_path] + + +def test_atomic_write_preserves_mode_on_success(tmp_path: Path) -> None: + config_path = tmp_path / "colima.yaml" + _write_config(config_path, "docker: {}\n") + config_path.chmod(0o640) + + colima_config.set_snapshotter(str(config_path), False) + + assert os.stat(config_path).st_mode & 0o777 == 0o640 diff --git a/tests/lib/test_colima_storage.py b/tests/lib/test_colima_storage.py new file mode 100644 index 00000000..f94048c0 --- /dev/null +++ b/tests/lib/test_colima_storage.py @@ -0,0 +1,705 @@ +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from unittest import mock + +import pytest + +from devenv import constants +from devenv.lib import colima +from devenv.lib import colima_config + + +@pytest.fixture(autouse=True) +def _default_lima_home() -> Iterator[None]: + with mock.patch.dict(constants.user_environ, {"LIMA_HOME": ""}): + yield + + +def _profile( + tmp_path: Path, config: str = "disk: 512\nruntime: docker\ndocker: {}\n" +) -> tuple[Path, Path]: + home = tmp_path / "home" + instance = home / ".colima/_lima/colima" + instance.mkdir(parents=True) + config_path = home / ".colima/default/colima.yaml" + config_path.parent.mkdir(parents=True) + config_path.write_text(config) + return home, config_path + + +def _prepare( + tmp_path: Path, + status: colima.ColimaStatus, + config: str = "disk: 512\nruntime: docker\ndocker: {}\n", + *, + disk: str = "", + store: str | None = None, + docker_info: str = "", + docker_error: str | None = None, +) -> tuple[Path, Path, mock.Mock]: + home, config_path = _profile(tmp_path, config) + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + store_path = home / ".colima/_store/colima.json" + if store is not None: + store_path.parent.mkdir(parents=True) + store_path.write_text(store) + + commands: list[tuple[str, ...]] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + commands.append(command) + if command[0] == "docker": + if docker_error: + raise RuntimeError(docker_error) + return docker_info + if command[0].endswith("limactl"): + return disk + raise AssertionError(command) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + ): + existing = colima._prepare_docker_store(status) + + return ( + config_path, + store_path, + mock.Mock(policy=existing, commands=commands), + ) + + +def test_missing_vm_does_not_add_legacy_override(tmp_path: Path) -> None: + home = tmp_path / "home" + (home / ".colima/default").mkdir(parents=True) + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + run.assert_not_called() + assert policy.existing_instance is False + assert policy.expected_snapshotter is True + + +def test_new_vm_persists_containerd_choice_in_existing_config( + tmp_path: Path, +) -> None: + home = tmp_path / "home" + config_path = home / ".colima/default/colima.yaml" + config_path.parent.mkdir(parents=True) + config_path.write_text( + "disk: 512\nruntime: docker\ndocker:\n log-driver: local\n" + ) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + run.assert_not_called() + assert policy == colima._StoragePolicy( + existing_instance=False, expected_snapshotter=True + ) + assert "log-driver: local" in config_path.read_text() + assert "containerd-snapshotter: true" in config_path.read_text() + + +@pytest.mark.parametrize("snapshotter", (True, False)) +def test_new_vm_preserves_explicit_store_choice( + tmp_path: Path, snapshotter: bool +) -> None: + home = tmp_path / "home" + config_path = home / ".colima/default/colima.yaml" + config_path.parent.mkdir(parents=True) + config_path.write_text( + "disk: 512\nruntime: docker\ndocker:\n features:\n" + f" containerd-snapshotter: {str(snapshotter).lower()}\n" + ) + original = config_path.read_bytes() + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + run.assert_not_called() + assert policy == colima._StoragePolicy( + existing_instance=False, expected_snapshotter=snapshotter + ) + assert config_path.read_bytes() == original + + +@pytest.mark.parametrize("snapshotter", (True, False)) +def test_new_vm_preserves_explicit_template_store_choice( + tmp_path: Path, snapshotter: bool +) -> None: + home = tmp_path / "home" + template_path = home / ".colima/_templates/default.yaml" + template_path.parent.mkdir(parents=True) + template_path.write_text( + "runtime: docker\ndocker:\n features:\n" + f" containerd-snapshotter: {str(snapshotter).lower()}\n" + ) + original = template_path.read_bytes() + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + run.assert_not_called() + assert policy == colima._StoragePolicy( + existing_instance=False, expected_snapshotter=snapshotter + ) + assert template_path.read_bytes() == original + + +def test_new_vm_rejects_storage_choice_colima_would_ignore( + tmp_path: Path, +) -> None: + home = tmp_path / "home" + config_path = home / ".colima/default/colima.yaml" + config_path.parent.mkdir(parents=True) + config_path.write_text( + "docker:\n features:\n containerd-snapshotter: false\n" + ) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + pytest.raises(colima_config.StorageMigrationBlocked, match="runtime"), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + run.assert_not_called() + + +def test_fresh_vm_containerd_store_survives_stop_and_start( + tmp_path: Path, +) -> None: + home = tmp_path / "home" + root = tmp_path / "sentry-devenv" + instance = home / ".colima/_lima/colima" + config_path = home / ".colima/default/colima.yaml" + statuses = iter( + ( + colima.ColimaStatus.DOWN, + colima.ColimaStatus.DOWN, + colima.ColimaStatus.DOWN, + ) + ) + starts: list[tuple[colima._StartMode, bool]] = [] + commands: list[tuple[str, ...]] = [] + + def start_vm( + mode: colima._StartMode, + status: colima.ColimaStatus | None = None, + *, + existing_instance: bool = False, + ) -> colima.ColimaStatus: + starts.append((mode, existing_instance)) + if not instance.exists(): + instance.mkdir(parents=True) + config_path.parent.mkdir(parents=True) + config_path.write_text("disk: 512\nruntime: docker\ndocker: {}\n") + return colima.ColimaStatus.UP + + def run(command: tuple[str, ...], **kwargs: object) -> str: + commands.append(command) + return 'overlayfs|[["driver-type","io.containerd.snapshotter.v1"]]' + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch.object(colima, "check", side_effect=lambda: next(statuses)), + mock.patch.object(colima, "_start", side_effect=start_vm), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + ): + assert colima.start() == colima.ColimaStatus.UP + first_config = config_path.read_bytes() + assert "containerd-snapshotter: true" in first_config.decode() + + assert colima.stop() == colima.ColimaStatus.DOWN + assert colima.start() == colima.ColimaStatus.UP + + assert starts == [ + (colima._StartMode.ENSURE_RUNNING, False), + (colima._StartMode.ENSURE_RUNNING, True), + ] + assert sum(command[0] == "docker" for command in commands) == 2 + assert ("colima", "stop") in commands + assert config_path.read_bytes() == first_config + + +@pytest.mark.parametrize("instance_kind", ("symlink", "file")) +def test_unexpected_instance_path_blocks_migration( + tmp_path: Path, instance_kind: str +) -> None: + home = tmp_path / "home" + instance = home / ".colima/_lima/colima" + instance.parent.mkdir(parents=True) + if instance_kind == "symlink": + target = tmp_path / "independent-vm" + target.mkdir() + instance.symlink_to(target) + else: + instance.write_text("not a VM directory") + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + run.assert_not_called() + + +@pytest.mark.parametrize("snapshotter", (True, False)) +def test_explicit_user_store_choice_is_not_overwritten( + tmp_path: Path, snapshotter: bool +) -> None: + config_path = ( + "disk: 512\nruntime: docker\ndocker:\n features:\n" + f" containerd-snapshotter: {str(snapshotter).lower()}\n" + ) + home, path = _profile(tmp_path, config_path) + original = path.read_bytes() + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch("devenv.lib.colima.proc.run") as run, + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + run.assert_not_called() + assert policy.existing_instance is True + assert policy.expected_snapshotter is snapshotter + assert path.read_bytes() == original + + +def test_stopped_legacy_vm_sets_overlay2_before_start(tmp_path: Path) -> None: + config_path, store_path, result = _prepare( + tmp_path, colima.ColimaStatus.DOWN + ) + + assert result.policy.existing_instance is True + assert result.policy.expected_snapshotter is False + assert "containerd-snapshotter: false" in config_path.read_text() + assert not store_path.exists() + assert result.commands == [ + ( + str(tmp_path / "sentry-devenv/bin/limactl"), + "disk", + "list", + "--json", + "colima", + ) + ] + + +def test_stopped_legacy_vm_ignores_standalone_lima_disk(tmp_path: Path) -> None: + home, config_path = _profile(tmp_path) + standalone_disk = home / ".lima/_disks/colima" + standalone_disk.mkdir(parents=True) + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + probe_environments: list[dict[str, str] | None] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + if command[0].endswith("limactl"): + env = kwargs.get("env") + probe_environments.append(env if isinstance(env, dict) else None) + lima_home = ( + env.get("LIMA_HOME", f"{home}/.lima") + if isinstance(env, dict) + else f"{home}/.lima" + ) + return '{"name":"colima"}' if lima_home == f"{home}/.lima" else "" + raise AssertionError(command) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + assert policy.expected_snapshotter is False + assert "containerd-snapshotter: false" in config_path.read_text() + assert probe_environments == [{"LIMA_HOME": f"{home}/.colima/_lima"}] + + +def test_stopped_modern_vm_disk_prevents_overlay2_override( + tmp_path: Path, +) -> None: + home, config_path = _profile(tmp_path) + original = config_path.read_bytes() + modern_disk = home / ".colima/_lima/_disks/colima" + modern_disk.parent.mkdir(parents=True) + modern_disk.touch() + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + probe_environments: list[dict[str, str] | None] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + if command[0].endswith("limactl"): + env = kwargs.get("env") + probe_environments.append(env if isinstance(env, dict) else None) + lima_home = ( + env.get("LIMA_HOME", f"{home}/.lima") + if isinstance(env, dict) + else f"{home}/.lima" + ) + return ( + '{"name":"colima"}' + if lima_home == f"{home}/.colima/_lima" + else "" + ) + raise AssertionError(command) + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + assert config_path.read_bytes() == original + assert probe_environments == [{"LIMA_HOME": f"{home}/.colima/_lima"}] + + +def test_explicit_lima_home_is_used_for_colima_instance_and_disk_probe( + tmp_path: Path, +) -> None: + home, config_path = _profile(tmp_path) + default_instance = home / ".colima/_lima/colima" + default_instance.rmdir() + explicit_lima_home = tmp_path / "custom-lima-home" + (explicit_lima_home / "colima").mkdir(parents=True) + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + probe_environments: list[dict[str, str] | None] = [] + + def run(command: tuple[str, ...], **kwargs: object) -> str: + if command[0].endswith("limactl"): + env = kwargs.get("env") + probe_environments.append(env if isinstance(env, dict) else None) + return "" + raise AssertionError(command) + + with ( + mock.patch.dict( + constants.user_environ, {"LIMA_HOME": str(explicit_lima_home)} + ), + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + ): + policy = colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + assert policy.expected_snapshotter is False + assert "containerd-snapshotter: false" in config_path.read_text() + assert probe_environments == [{"LIMA_HOME": str(explicit_lima_home)}] + + +def test_running_overlay2_vm_is_probed_through_colima_context( + tmp_path: Path, +) -> None: + custom = ( + "disk: 512\nruntime: docker\ndocker:\n" + " log-driver: json-file\n features:\n buildkit: false\n" + ) + config_path, _, result = _prepare( + tmp_path, colima.ColimaStatus.UP, custom, docker_info="overlay2|[]" + ) + + assert "containerd-snapshotter: false" in config_path.read_text() + assert result.policy.expected_snapshotter is False + assert "log-driver: json-file" in config_path.read_text() + assert "buildkit: false" in config_path.read_text() + assert result.commands == [ + ( + "docker", + "--context=colima", + "info", + "--format", + "{{.Driver}}|{{json .DriverStatus}}", + ) + ] + + +def test_unhealthy_legacy_vm_uses_positive_disk_evidence( + tmp_path: Path, +) -> None: + config_path, _, result = _prepare( + tmp_path, + colima.ColimaStatus.UNHEALTHY, + docker_error="daemon unavailable", + ) + + assert "containerd-snapshotter: false" in config_path.read_text() + assert result.policy.expected_snapshotter is False + assert result.commands == [ + ( + "docker", + "--context=colima", + "info", + "--format", + "{{.Driver}}|{{json .DriverStatus}}", + ), + ( + str(tmp_path / "sentry-devenv/bin/limactl"), + "disk", + "list", + "--json", + "colima", + ), + ] + + +def test_unhealthy_vm_with_data_disk_blocks_without_backend_probe( + tmp_path: Path, +) -> None: + home, config_path = _profile(tmp_path) + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + original = config_path.read_bytes() + + def run(command: tuple[str, ...], **kwargs: object) -> str: + if command[0] == "docker": + raise RuntimeError("daemon unavailable") + return '{"name":"colima"}' + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.UNHEALTHY) + + assert config_path.read_bytes() == original + + +def test_running_containerd_vm_is_preserved(tmp_path: Path) -> None: + config_path, _, _ = _prepare( + tmp_path, + colima.ColimaStatus.UP, + docker_info='overlayfs|[["driver-type","io.containerd.snapshotter.v1"]]', + ) + + assert "containerd-snapshotter: true" in config_path.read_text() + + +def test_running_overlay2_vm_with_empty_driver_status_is_preserved( + tmp_path: Path, +) -> None: + config_path, _, result = _prepare( + tmp_path, colima.ColimaStatus.UP, docker_info="overlay2|null" + ) + + assert "containerd-snapshotter: false" in config_path.read_text() + assert result.policy.expected_snapshotter is False + + +def test_started_migration_verifies_the_colima_docker_backend() -> None: + policy = colima._StoragePolicy( + existing_instance=True, expected_snapshotter=False + ) + with mock.patch( + "devenv.lib.colima.proc.run", return_value="overlay2|[]" + ) as run: + assert ( + colima._verify_storage_after_start( + colima.ColimaStatus.UP, policy, vm_was_started=True + ) + == colima.ColimaStatus.UP + ) + + assert run.call_args.args[0][:3] == ("docker", "--context=colima", "info") + + +def test_saved_overlay2_choice_is_verified_after_restart( + tmp_path: Path, +) -> None: + config = ( + "disk: 512\nruntime: docker\ndocker:\n features:\n" + " containerd-snapshotter: false\n" + ) + config_path, _, result = _prepare( + tmp_path, colima.ColimaStatus.DOWN, config + ) + + assert result.policy.expected_snapshotter is False + with mock.patch( + "devenv.lib.colima.proc.run", return_value="overlay2|[]" + ) as run: + assert ( + colima._verify_storage_after_start( + colima.ColimaStatus.UP, result.policy, vm_was_started=True + ) + == colima.ColimaStatus.UP + ) + + assert "containerd-snapshotter: false" in config_path.read_text() + assert run.call_args.args[0][:3] == ("docker", "--context=colima", "info") + + +def test_started_migration_blocks_if_colima_selected_another_backend( + capsys: pytest.CaptureFixture[str], +) -> None: + policy = colima._StoragePolicy( + existing_instance=True, expected_snapshotter=False + ) + with mock.patch( + "devenv.lib.colima.proc.run", + return_value='overlayfs|[["driver-type","io.containerd.snapshotter.v1"]]', + ) as run: + assert ( + colima._verify_storage_after_start( + colima.ColimaStatus.UP, policy, vm_was_started=True + ) + == colima.ColimaStatus.UNHEALTHY + ) + + assert run.call_args.args[0][:3] == ("docker", "--context=colima", "info") + assert "Images and containers may be hidden" in capsys.readouterr().out + + +def test_healthy_start_skips_redundant_backend_probe() -> None: + policy = colima._StoragePolicy( + existing_instance=True, expected_snapshotter=False + ) + with mock.patch("devenv.lib.colima.proc.run") as run: + assert ( + colima._verify_storage_after_start( + colima.ColimaStatus.UP, policy, vm_was_started=False + ) + == colima.ColimaStatus.UP + ) + run.assert_not_called() + + +@pytest.mark.parametrize( + ("store", "disk", "config"), + ( + ( + '{"disk_formatted":true,"disk_runtime":"docker"}', + '{"name":"colima"}', + "disk: 512\nruntime: docker\ndocker: {}\n", + ), + ( + '{"disk_formatted":false,"disk_runtime":""}', + '{"name":"colima"}', + "disk: 512\nruntime: docker\ndocker: {}\n", + ), + ( + '{"disk_formatted":false,"disk_runtime":""}', + '{"name":""}', + "disk: 512\nrootDisk: 20\nruntime: docker\ndocker: {}\n", + ), + ( + '{"disk_formatted":false,"disk_runtime":""}', + '{"name":""}', + "disk: 100\nruntime: docker\ndocker: {}\n", + ), + ), +) +def test_stopped_ambiguous_vm_blocks_without_rewriting( + tmp_path: Path, store: str, disk: str, config: str +) -> None: + home, config_path = _profile(tmp_path, config) + store_path = home / ".colima/_store/colima.json" + store_path.parent.mkdir(parents=True) + store_path.write_text(store) + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + original = config_path.read_bytes() + + def run(command: tuple[str, ...], **kwargs: object) -> str: + return disk + + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", side_effect=run), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + + assert config_path.read_bytes() == original + + +def test_corrupt_store_metadata_blocks_without_rewriting( + tmp_path: Path, +) -> None: + home, config_path = _profile(tmp_path) + store_path = home / ".colima/_store/colima.json" + store_path.parent.mkdir(parents=True) + store_path.write_text("not json") + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + original = config_path.read_bytes() + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + assert config_path.read_bytes() == original + + +def test_failed_running_backend_probe_blocks_without_rewriting( + tmp_path: Path, +) -> None: + home, config_path = _profile(tmp_path) + original = config_path.read_bytes() + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch( + "devenv.lib.colima.proc.run", + side_effect=RuntimeError("daemon unavailable"), + ), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.UP) + assert config_path.read_bytes() == original + + +def test_invalid_disk_json_blocks_without_rewriting(tmp_path: Path) -> None: + home, config_path = _profile(tmp_path) + original = config_path.read_bytes() + root = tmp_path / "sentry-devenv" + limactl = root / "bin/limactl" + limactl.parent.mkdir(parents=True) + limactl.touch() + with ( + mock.patch.object(colima, "home", str(home)), + mock.patch.object(colima, "root", str(root)), + mock.patch("devenv.lib.colima.proc.run", return_value="invalid"), + pytest.raises(colima_config.StorageMigrationBlocked), + ): + colima._prepare_docker_store(colima.ColimaStatus.DOWN) + assert config_path.read_bytes() == original diff --git a/tests/lib/test_limactl.py b/tests/lib/test_limactl.py new file mode 100644 index 00000000..d1ec082e --- /dev/null +++ b/tests/lib/test_limactl.py @@ -0,0 +1,535 @@ +from __future__ import annotations + +import os +import pathlib +import shutil +import tarfile +from unittest.mock import patch + +import pytest + +from devenv import constants +from devenv.lib import limactl + +_RELEASE_FILES = ( + "bin/lima", + "bin/limactl", + "libexec/lima/limactl-mcp", + "libexec/lima/lima-driver-krunkit", + "share/lima/lima-guestagent.Linux-aarch64.gz", + "share/lima/lima-guestagent.Linux-x86_64.gz", + "share/lima/templates/default.yaml", +) + + +def _release( + tmp_path: pathlib.Path, *, omit: str | None = None +) -> pathlib.Path: + source = tmp_path / "release" + shutil.rmtree(source, ignore_errors=True) + for name in _RELEASE_FILES: + if name == omit: + continue + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "#!/bin/sh\nexit 0\n" if name.startswith("bin/") else name + ) + if name.startswith(("bin/", "libexec/")): + path.chmod(0o755) + archive_path = tmp_path / "lima.tar.gz" + with tarfile.open(archive_path, "w:gz") as tarf: + tarf.add(source, arcname=".") + return archive_path + + +def _install_from_release( + tmp_path: pathlib.Path, binroot: str, machine: str = "darwin_arm64" +) -> None: + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch("devenv.lib.archive.download", side_effect=download), + patch.object(limactl, "SYSTEM_MACHINE", machine), + ): + limactl._install("url", "sha256", binroot) + + +def _installed_version(command: tuple[str, ...], **kwargs: object) -> str: + return f"{pathlib.Path(command[0]).name} version 2.2.1" + + +def test_version_detection_uses_bundled_limactl(tmp_path: pathlib.Path) -> None: + prefix = tmp_path / "sentry-devenv" + binroot = prefix / "bin" + binroot.mkdir(parents=True) + lima = binroot / "lima" + lima.write_text( + '#!/bin/sh\n: "${LIMACTL:=limactl}"\nexec "$LIMACTL" "$@"\n' + ) + lima.chmod(0o755) + limactl_binary = binroot / "limactl" + limactl_binary.write_text('#!/bin/sh\nprintf "limactl version 2.2.1\\n"\n') + limactl_binary.chmod(0o755) + host_limactl = tmp_path / "host-limactl" + host_limactl.write_text('#!/bin/sh\nprintf "limactl version 2.1.0\\n"\n') + host_limactl.chmod(0o755) + + with patch.dict(constants.user_environ, {"LIMACTL": str(host_limactl)}): + assert limactl._staged_version(str(prefix)) == "2.2.1" + assert limactl._installed_version(str(binroot)) == "2.2.1" + + +def test_install_keeps_native_lima_layout(tmp_path: pathlib.Path) -> None: + prefix = tmp_path / "sentry-devenv" + binroot = str(prefix / "bin") + + _install_from_release(tmp_path, binroot) + + assert (prefix / "bin/lima").is_file() + assert (prefix / "bin/limactl").is_file() + assert (prefix / "share/lima/lima-guestagent.Linux-aarch64.gz").is_file() + assert (prefix / "libexec/lima/lima-driver-krunkit").is_file() + assert sorted(path.name for path in (prefix / "bin").iterdir()) == [ + "lima", + "limactl", + ] + + +def test_install_removes_old_lima_layout_from_devenv_prefix( + tmp_path: pathlib.Path, +) -> None: + prefix = tmp_path / "sentry-devenv" + binroot = prefix / "bin" + old_paths = ( + binroot / "templates/default.yaml", + binroot / "lima-guestagent.Linux-aarch64", + binroot / "lima-guestagent.Linux-x86_64", + prefix / "share/lima/old-guestagent", + prefix / "libexec/lima/old-driver", + ) + for path in old_paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"old layout") + + _install_from_release(tmp_path, str(binroot)) + + assert not (binroot / "templates").exists() + assert not (binroot / "lima-guestagent.Linux-aarch64").exists() + assert not (binroot / "lima-guestagent.Linux-x86_64").exists() + assert not (prefix / "share/lima/old-guestagent").exists() + assert not (prefix / "libexec/lima/old-driver").exists() + assert (prefix / "share/lima/templates/default.yaml").is_file() + + +@pytest.mark.parametrize( + ("machine", "agent", "driver"), + ( + ( + "darwin_arm64", + "lima-guestagent.Linux-aarch64.gz", + "lima-driver-krunkit", + ), + ("darwin_x86_64", "lima-guestagent.Linux-x86_64.gz", "limactl-mcp"), + ), +) +def test_install_global_installs_native_payload( + tmp_path: pathlib.Path, machine: str, agent: str, driver: str | None +) -> None: + root = str(tmp_path / "sentry-devenv") + binroot = f"{root}/bin" + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "root", root), + patch.object(limactl, "SYSTEM_MACHINE", machine), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + ): + limactl.install_global() + + assert os.path.isfile(f"{root}/share/lima/{agent}") + assert os.path.isfile(f"{binroot}/limactl") + if driver: + assert os.path.isfile(f"{root}/libexec/lima/{driver}") + + +def test_install_global_does_not_require_optional_intel_mcp_plugin( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + _install_from_release(tmp_path, str(root / "bin"), machine="darwin_x86_64") + (root / "libexec/lima/limactl-mcp").unlink() + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_x86_64"), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + patch.object(limactl, "_install") as install, + ): + limactl.install_global() + + install.assert_not_called() + + +def test_uninstall_removes_lima_one_files_and_preserves_user_data( + tmp_path: pathlib.Path, +) -> None: + prefix = tmp_path / "sentry-devenv" + binroot = prefix / "bin" + home = tmp_path / "home" + user_data = home / ".lima/sentinel" + user_data.parent.mkdir(parents=True) + user_data.write_bytes(b"standalone lima vm\x00data") + original = user_data.read_bytes() + binroot.mkdir(parents=True) + for name in ( + "lima", + "limactl", + "lima-guestagent.Linux-aarch64", + "lima-guestagent.Linux-x86_64", + "templates/default.yaml", + "colima", + ): + path = binroot / name + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + with patch.object(limactl, "home", str(home), create=True): + limactl.uninstall(str(binroot)) + + assert sorted(path.name for path in binroot.iterdir()) == ["colima"] + assert user_data.read_bytes() == original + + +@pytest.mark.parametrize( + "missing", + ( + "bin/lima", + "bin/limactl", + "share/lima/lima-guestagent.Linux-aarch64.gz", + "share/lima/templates/default.yaml", + "libexec/lima/lima-driver-krunkit", + ), +) +def test_install_global_repairs_missing_payload( + tmp_path: pathlib.Path, missing: str, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "sentry-devenv" + home = tmp_path / "home" + sentinel = home / ".lima/sentinel" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"standalone lima vm\x00data") + original = sentinel.read_bytes() + colima_disk = home / ".colima/_lima/colima/diffdisk" + colima_disk.parent.mkdir(parents=True) + colima_disk.write_bytes(b"Colima VM data") + original_colima_disk = colima_disk.read_bytes() + _install_from_release(tmp_path, str(root / "bin")) + (root / missing).unlink() + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch.object(limactl, "home", str(home), create=True), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + ): + limactl.install_global() + + assert (root / missing).is_file() + assert sentinel.read_bytes() == original + assert colima_disk.read_bytes() == original_colima_disk + assert missing in capsys.readouterr().out + + +def test_install_global_repairs_corrupt_guest_agent( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + _install_from_release(tmp_path, str(root / "bin")) + guest_agent = root / "share/lima/lima-guestagent.Linux-aarch64.gz" + guest_agent.write_bytes(b"") + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + ): + limactl.install_global() + + assert guest_agent.stat().st_size > 0 + + +def test_install_global_repairs_non_executable_driver( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + _install_from_release(tmp_path, str(root / "bin")) + driver = root / "libexec/lima/lima-driver-krunkit" + driver.chmod(0o644) + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + ): + limactl.install_global() + + assert os.access(driver, os.X_OK) + + +def test_install_global_reinstalls_wrong_version( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + home = tmp_path / "home" + sentinel = home / ".lima/sentinel" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"standalone lima vm\x00data") + original = sentinel.read_bytes() + _install_from_release(tmp_path, str(root / "bin")) + release = _release(tmp_path) + + def installed_version(command: tuple[str, ...], **kwargs: object) -> str: + version = "2.1.0" if downloads == 0 else "2.2.1" + return f"{pathlib.Path(command[0]).name} version {version}" + + downloads = 0 + + def download(url: str, sha256: str, dest: str) -> str: + nonlocal downloads + downloads += 1 + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch.object(limactl, "home", str(home), create=True), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=installed_version), + ): + limactl.install_global() + + assert (root / "bin/limactl").is_file() + assert sentinel.read_bytes() == original + + +def test_install_global_reinstalls_mismatched_lima_commands( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + _install_from_release(tmp_path, str(root / "bin")) + release = _release(tmp_path) + downloads = 0 + + def download(url: str, sha256: str, dest: str) -> str: + nonlocal downloads + downloads += 1 + shutil.copy(release, dest) + return dest + + def installed_version(command: tuple[str, ...], **kwargs: object) -> str: + command_version = "2.2.1" + if downloads == 0 and pathlib.Path(command[0]).name == "lima": + command_version = "2.2.0" + return f"{pathlib.Path(command[0]).name} version {command_version}" + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=installed_version), + ): + limactl.install_global() + + assert downloads == 1 + assert (root / "bin/lima").is_file() + assert (root / "bin/limactl").is_file() + + +def test_install_global_is_idempotent(tmp_path: pathlib.Path) -> None: + root = tmp_path / "sentry-devenv" + _install_from_release(tmp_path, str(root / "bin")) + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.limactl.proc.run", side_effect=_installed_version), + patch.object(limactl, "_install") as install, + ): + limactl.install_global() + + install.assert_not_called() + + +def test_reinstall_and_uninstall_preserve_lima_user_data( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + home = tmp_path / "home" + sentinel = home / ".lima/sentinel" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"standalone lima vm\x00data") + original = sentinel.read_bytes() + _install_from_release(tmp_path, str(root / "bin")) + release = _release(tmp_path) + + downloads = 0 + + def download(url: str, sha256: str, dest: str) -> str: + nonlocal downloads + downloads += 1 + shutil.copy(release, dest) + return dest + + def installed_version(command: tuple[str, ...], **kwargs: object) -> str: + version = "2.1.0" if downloads == 0 else "2.2.1" + return f"{pathlib.Path(command[0]).name} version {version}" + + with ( + patch.object(limactl, "root", str(root)), + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch.object(limactl, "home", str(home), create=True), + patch("devenv.lib.archive.download", side_effect=download), + patch("devenv.lib.limactl.proc.run", side_effect=installed_version), + ): + limactl.install_global() + limactl.uninstall(f"{root}/bin") + + assert sentinel.read_bytes() == original + + +def test_interrupted_download_preserves_lima_install_and_user_data( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + home = tmp_path / "home" + sentinel = home / ".lima/sentinel" + sentinel.parent.mkdir(parents=True) + sentinel.write_bytes(b"standalone lima vm\x00data") + original = sentinel.read_bytes() + binroot = root / "bin" + _install_from_release(tmp_path, str(binroot)) + lima_binary = (binroot / "limactl").read_bytes() + + with ( + patch.object(limactl, "home", str(home), create=True), + patch( + "devenv.lib.archive.download", + side_effect=RuntimeError("download interrupted"), + ), + pytest.raises(RuntimeError, match="download interrupted"), + ): + limactl._install("url", "sha256", str(binroot)) + + assert sentinel.read_bytes() == original + assert (binroot / "limactl").read_bytes() == lima_binary + + +def test_incomplete_release_preserves_previous_lima_install( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + binroot = root / "bin" + _install_from_release(tmp_path, str(binroot)) + previous = { + path: path.read_bytes() + for path in ( + binroot / "lima", + binroot / "limactl", + root / "share/lima/lima-guestagent.Linux-aarch64.gz", + root / "share/lima/templates/default.yaml", + root / "libexec/lima/lima-driver-krunkit", + ) + } + release = _release(tmp_path, omit="bin/limactl") + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + with ( + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.archive.download", side_effect=download), + pytest.raises(RuntimeError, match="release is incomplete"), + ): + limactl._install("url", "sha256", str(binroot)) + + assert {path: path.read_bytes() for path in previous} == previous + + +def test_failed_payload_replace_restores_previous_lima_install( + tmp_path: pathlib.Path, +) -> None: + root = tmp_path / "sentry-devenv" + binroot = root / "bin" + _install_from_release(tmp_path, str(binroot)) + previous = { + path: path.read_bytes() + for path in ( + binroot / "lima", + binroot / "limactl", + root / "share/lima/lima-guestagent.Linux-aarch64.gz", + root / "share/lima/templates/default.yaml", + root / "libexec/lima/lima-driver-krunkit", + ) + } + release = _release(tmp_path) + + def download(url: str, sha256: str, dest: str) -> str: + shutil.copy(release, dest) + return dest + + real_replace = os.replace + failed = False + + def fail_payload_replace(source: str, destination: str) -> None: + nonlocal failed + if ( + not failed + and str(source).startswith(f"{binroot}/") + and str(source).endswith("/libexec/lima") + ): + failed = True + raise OSError("payload move failed") + real_replace(source, destination) + + with ( + patch.object(limactl, "SYSTEM_MACHINE", "darwin_arm64"), + patch("devenv.lib.archive.download", side_effect=download), + patch( + "devenv.lib.limactl.os.replace", side_effect=fail_payload_replace + ), + pytest.raises(OSError, match="payload move failed"), + ): + limactl._install("url", "sha256", str(binroot)) + + assert failed + assert {path: path.read_bytes() for path in previous} == previous diff --git a/tests/lib/test_proc.py b/tests/lib/test_proc.py index 0ef6a606..26da4595 100644 --- a/tests/lib/test_proc.py +++ b/tests/lib/test_proc.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import os from unittest.mock import patch @@ -116,3 +117,16 @@ def test_run_command_failed_capture_combined_output() -> None: """ ) + + +def test_run_flushes_prints_before_the_command( + capfd: pytest.CaptureFixture[str], +) -> None: + # like stdout in CI logs or when piped: not a tty, so block buffered + stdout = io.TextIOWrapper(os.fdopen(os.dup(1), "wb")) + with patch("sys.stdout", stdout): + print("before") + run(("echo", "during")) + stdout.flush() + + assert capfd.readouterr().out == "before\nduring\n"