feat(qemu): back the guest boot image with an overlay by default

.github/workflows/verify_and_publish.yml

@@ -75,6 +75,13 @@ jobs:
         make info
         echo "$HOME/.local/bin" >> $GITHUB_PATH
 
+    # For the guest boot-image test that opens a real qcow2 overlay
+    - name: qemu-img
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt-get -qq update
+        sudo apt-get -qq install -y qemu-utils
+
     - name: All
       run: |
         make all

CHANGELOG.md

@@ -18,6 +18,19 @@ per command. The connection has a 30 second keepalive, is made again
 on next use when it has dropped, e.g. as the remote rebooted, and is closed by
 the new `Cijoe.close()`, at the end of a pytest session, or as Python exits.
 
+The qemu guest's boot image is by default a qcow2 overlay backed by the staged
+system image, instead of a copy of it, so a boot no longer starts with a
+multi-GB copy and guests from one image share it. `qemu.guest_initialize`
+takes `--overlay false` to copy as before; `diskimage_from_cloudimage` does so,
+since it must write the image it produces.
+
+The report options are now `--report-open` to also open the report in a
+browser, which is no longer done by default, `--report-skip` to not write it,
+and `--report-only` to write it for an existing output and exit.
+`-n/--no-report` and `-p/--produce-report` remain as deprecated spellings of
+`--report-skip` and `--report-only`, and `-s/--skip-report` is accepted and
+ignored, each with a warning.
+
 ## 0.9.59
 
 Renamed the "workflow" concept to "task" across the codebase, docs, and example

src/cijoe/qemu/scripts/guest_initialize.py

@@ -14,6 +14,10 @@
 
 Additionally, the script prepares guest storage, including the boot drive, 
 using an existing `.qcow2` image (e.g., created via Cloud-init, Packer, etc.).
+The boot drive is a qcow2 overlay backed by that image: the system-imaging
+store holds the images, and each guest references one, holding only what it
+writes. The image must then stay unchanged while a guest uses it; with
+``overlay`` set to false, the boot drive is a copy of the image instead.
 
 Configuration
 -------------
@@ -33,7 +37,7 @@
 """
 import errno
 import logging as log
-from argparse import ArgumentParser
+from argparse import Action, ArgumentParser
 from pathlib import Path
 
 from cijoe.core.misc import download, download_and_verify
@@ -48,6 +52,18 @@ def add_args(parser: ArgumentParser):
         help="Name of the system image. This will overwrite any system image name defined in the configuration file.",
     )
 
+    class StringToBoolAction(Action):
+        def __call__(self, parser, namespace, values, option_string=None):
+            setattr(namespace, self.dest, values == "true")
+
+    parser.add_argument(
+        "--overlay",
+        choices=["true", "false"],
+        default=True,
+        action=StringToBoolAction,
+        help="Back the boot image with the disk image, as a qcow2 overlay; with false, copy it. With the overlay, the disk image must stay unchanged while the guest is in use.",
+    )
+
 
 def main(args, cijoe):
     """Provision using an existing boot image"""
@@ -93,7 +109,7 @@ def main(args, cijoe):
             log.error(f"err({err}, path({path})")
             return err
 
-    err = guest.initialize(diskimage_path)
+    err = guest.initialize(diskimage_path, overlay=args.overlay)
     if err:
         log.error(f"guest.initialize({diskimage_path}); err({err})")
         return err

src/cijoe/qemu/wrapper.py

@@ -111,12 +111,26 @@ def get_pid(self):
 
         return pid
 
-    def initialize(self, diskimage_path: Optional[Path] = None):
-        """Create a 'home' for the guest'"""
+    def initialize(self, diskimage_path: Optional[Path] = None, overlay=True):
+        """Create a 'home' for the guest'
+
+        The boot image is a qcow2 overlay backed by the disk image, so the disk
+        image must stay unchanged for as long as the guest is used. Without
+        'overlay', the boot image is a copy of the disk image instead.
+        """
 
         os.makedirs(self.guest_path, exist_ok=True)
 
-        if diskimage_path:
+        if diskimage_path and overlay:
+            backing = Path(diskimage_path).resolve()
+            err, _ = qemu_img(
+                self.cijoe,
+                f"create -f qcow2 -F qcow2 -b {backing} {self.boot_img}",
+            )
+            if err:
+                log.error(f"Failed creating overlay on({backing}); err({err})")
+                return err
+        elif diskimage_path:
             err, _ = self.cijoe.run_local(f"cp {diskimage_path} {self.boot_img}")
             if err:
                 log.error(f"Failed copying diskimage({diskimage_path}); err({err})")

src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py

@@ -110,7 +110,8 @@ def diskimage_from_cloudimage(cijoe, image: dict):
 
     guest = Guest(cijoe, cijoe.config, guest_name)
     guest.kill()  # Ensure the guest is *not* running
-    guest.initialize(cloud_image_path)  # Initialize using the cloudimage
+    # A copy, not an overlay: boot.img becomes the disk image further down
+    guest.initialize(cloud_image_path, overlay=False)
 
     # Create seed.img, with data and meta embedded
     guest_metadata_path = guest.guest_path / "meta-data"

tests/qemu/test_guest_initialize.py

@@ -0,0 +1,87 @@
+"""Unit tests for the boot image that Guest.initialize prepares.
+
+A real Cijoe from a real Config, as in the other tests; the command-level tests
+record what run_local would run instead of running it, so they need no qemu. The
+last one creates real qcow2 images, and is skipped where qemu-img is not
+installed.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from cijoe.core.command import Cijoe
+from cijoe.core.resources import Config
+from cijoe.qemu.wrapper import Guest
+
+
+def make_guest(tmp_path, monkeypatch=None):
+    """A Guest named 'g' under tmp_path; with monkeypatch, its commands are recorded"""
+
+    config = Config(Path("guest.toml"))
+    config.options = {
+        "qemu": {
+            "img_bin": "qemu-img",
+            "guests": {"g": {"path": str(tmp_path / "guest")}},
+        }
+    }
+    cijoe = Cijoe(config, tmp_path / "output", False)
+    cijoe.set_output_ident("test")
+
+    commands = []
+
+    def record(cmd, *args, **kwargs):
+        commands.append(cmd)
+        return 0, None
+
+    if monkeypatch is not None:
+        monkeypatch.setattr(cijoe, "run_local", record)
+
+    return Guest(cijoe, config, "g"), commands
+
+
+def test_copy_without_overlay(tmp_path, monkeypatch):
+    guest, commands = make_guest(tmp_path, monkeypatch)
+
+    assert guest.initialize(tmp_path / "disk.qcow2", overlay=False) == 0
+    assert commands == [f"cp {tmp_path / 'disk.qcow2'} {guest.boot_img}"]
+
+
+def test_overlay_by_default(tmp_path, monkeypatch):
+    guest, commands = make_guest(tmp_path, monkeypatch)
+
+    assert guest.initialize(tmp_path / "disk.qcow2") == 0
+    assert commands == [
+        f"qemu-img create -f qcow2 -F qcow2 -b {(tmp_path / 'disk.qcow2').resolve()}"
+        f" {guest.boot_img}"
+    ]
+
+
+def test_overlay_backing_path_is_absolute(tmp_path, monkeypatch):
+    """qemu reads a relative backing path from the overlay's directory, not the cwd"""
+
+    monkeypatch.chdir(tmp_path)
+    guest, commands = make_guest(tmp_path, monkeypatch)
+
+    assert guest.initialize("disk.qcow2") == 0
+    assert f"-b {tmp_path.resolve() / 'disk.qcow2'} " in commands[0]
+
+
+@pytest.mark.skipif(shutil.which("qemu-img") is None, reason="needs qemu-img")
+def test_overlay_is_a_qcow2_backed_by_the_disk_image(tmp_path):
+    disk = tmp_path / "disk.qcow2"
+    subprocess.run(["qemu-img", "create", "-f", "qcow2", str(disk), "64M"], check=True)
+
+    guest, _ = make_guest(tmp_path)
+    assert guest.initialize(disk) == 0
+
+    info = subprocess.run(
+        ["qemu-img", "info", "--output=json", str(guest.boot_img)],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    assert f'"backing-filename": "{disk.resolve()}"' in info
+    assert '"virtual-size": 67108864' in info