From 1fb81abb294aa98e0c683c6d882c03c76e7e90b8 Mon Sep 17 00:00:00 2001 From: "Simon A. F. Lund" Date: Fri, 23 Oct 2026 10:35:58 +0200 Subject: [PATCH 1/2] feat(qemu): back the guest boot image with an overlay by default guest_initialize copied the staged disk image to the guest's boot.img. For a large image that copy is a real cost: the 11.7 GiB qcow2 of a ROCm guest takes over a minute on a CI runner, which then throws the guest away, and every further guest from the same image pays it again, in time and in disk. boot.img is now a qcow2 overlay backed by the staged image, which takes no time, and several guests share one image, each holding only what it writes. That matches the split already there: system-imaging is the store of images, and a guest references one. The image must stay unchanged while a guest uses it. 'overlay: false' restores the copy, and diskimage_from_cloudimage asks for it, since it turns boot.img into the disk image afterwards. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Simon A. F. Lund --- .github/workflows/verify_and_publish.yml | 7 ++ CHANGELOG.md | 6 ++ src/cijoe/qemu/scripts/guest_initialize.py | 20 ++++- src/cijoe/qemu/wrapper.py | 20 ++++- .../scripts/diskimage_from_cloudimage.py | 3 +- tests/qemu/test_guest_initialize.py | 87 +++++++++++++++++++ 6 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 tests/qemu/test_guest_initialize.py diff --git a/.github/workflows/verify_and_publish.yml b/.github/workflows/verify_and_publish.yml index 8bda7ba9..abe54f68 100644 --- a/.github/workflows/verify_and_publish.yml +++ b/.github/workflows/verify_and_publish.yml @@ -75,6 +75,13 @@ jobs: make info echo "$HOME/.local/bin" >> $GITHUB_PATH + # For the guest boot-image test that opens a real qcow2 overlay + - name: qemu-img + if: runner.os == 'Linux' + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y qemu-utils + - name: All run: | make all diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a79f24d..39a8413e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,12 @@ per command. The connection has a 30 second keepalive, is made again on next use when it has dropped, e.g. as the remote rebooted, and is closed by the new `Cijoe.close()`, at the end of a pytest session, or as Python exits. +The qemu guest's boot image is by default a qcow2 overlay backed by the staged +system image, instead of a copy of it, so a boot no longer starts with a +multi-GB copy and guests from one image share it. `qemu.guest_initialize` +takes `--overlay false` to copy as before; `diskimage_from_cloudimage` does so, +since it must write the image it produces. + ## 0.9.59 Renamed the "workflow" concept to "task" across the codebase, docs, and example diff --git a/src/cijoe/qemu/scripts/guest_initialize.py b/src/cijoe/qemu/scripts/guest_initialize.py index 97986e42..cf4dab70 100644 --- a/src/cijoe/qemu/scripts/guest_initialize.py +++ b/src/cijoe/qemu/scripts/guest_initialize.py @@ -14,6 +14,10 @@ Additionally, the script prepares guest storage, including the boot drive, using an existing `.qcow2` image (e.g., created via Cloud-init, Packer, etc.). +The boot drive is a qcow2 overlay backed by that image: the system-imaging +store holds the images, and each guest references one, holding only what it +writes. The image must then stay unchanged while a guest uses it; with +``overlay`` set to false, the boot drive is a copy of the image instead. Configuration ------------- @@ -33,7 +37,7 @@ """ import errno import logging as log -from argparse import ArgumentParser +from argparse import Action, ArgumentParser from pathlib import Path from cijoe.core.misc import download, download_and_verify @@ -48,6 +52,18 @@ def add_args(parser: ArgumentParser): help="Name of the system image. This will overwrite any system image name defined in the configuration file.", ) + class StringToBoolAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values == "true") + + parser.add_argument( + "--overlay", + choices=["true", "false"], + default=True, + action=StringToBoolAction, + help="Back the boot image with the disk image, as a qcow2 overlay; with false, copy it. With the overlay, the disk image must stay unchanged while the guest is in use.", + ) + def main(args, cijoe): """Provision using an existing boot image""" @@ -93,7 +109,7 @@ def main(args, cijoe): log.error(f"err({err}, path({path})") return err - err = guest.initialize(diskimage_path) + err = guest.initialize(diskimage_path, overlay=args.overlay) if err: log.error(f"guest.initialize({diskimage_path}); err({err})") return err diff --git a/src/cijoe/qemu/wrapper.py b/src/cijoe/qemu/wrapper.py index ecd1f514..5f85497e 100644 --- a/src/cijoe/qemu/wrapper.py +++ b/src/cijoe/qemu/wrapper.py @@ -111,12 +111,26 @@ def get_pid(self): return pid - def initialize(self, diskimage_path: Optional[Path] = None): - """Create a 'home' for the guest'""" + def initialize(self, diskimage_path: Optional[Path] = None, overlay=True): + """Create a 'home' for the guest' + + The boot image is a qcow2 overlay backed by the disk image, so the disk + image must stay unchanged for as long as the guest is used. Without + 'overlay', the boot image is a copy of the disk image instead. + """ os.makedirs(self.guest_path, exist_ok=True) - if diskimage_path: + if diskimage_path and overlay: + backing = Path(diskimage_path).resolve() + err, _ = qemu_img( + self.cijoe, + f"create -f qcow2 -F qcow2 -b {backing} {self.boot_img}", + ) + if err: + log.error(f"Failed creating overlay on({backing}); err({err})") + return err + elif diskimage_path: err, _ = self.cijoe.run_local(f"cp {diskimage_path} {self.boot_img}") if err: log.error(f"Failed copying diskimage({diskimage_path}); err({err})") diff --git a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py index 40d4e3cf..716eb96d 100644 --- a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py +++ b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py @@ -110,7 +110,8 @@ def diskimage_from_cloudimage(cijoe, image: dict): guest = Guest(cijoe, cijoe.config, guest_name) guest.kill() # Ensure the guest is *not* running - guest.initialize(cloud_image_path) # Initialize using the cloudimage + # A copy, not an overlay: boot.img becomes the disk image further down + guest.initialize(cloud_image_path, overlay=False) # Create seed.img, with data and meta embedded guest_metadata_path = guest.guest_path / "meta-data" diff --git a/tests/qemu/test_guest_initialize.py b/tests/qemu/test_guest_initialize.py new file mode 100644 index 00000000..50225281 --- /dev/null +++ b/tests/qemu/test_guest_initialize.py @@ -0,0 +1,87 @@ +"""Unit tests for the boot image that Guest.initialize prepares. + +A real Cijoe from a real Config, as in the other tests; the command-level tests +record what run_local would run instead of running it, so they need no qemu. The +last one creates real qcow2 images, and is skipped where qemu-img is not +installed. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +from cijoe.core.command import Cijoe +from cijoe.core.resources import Config +from cijoe.qemu.wrapper import Guest + + +def make_guest(tmp_path, monkeypatch=None): + """A Guest named 'g' under tmp_path; with monkeypatch, its commands are recorded""" + + config = Config(Path("guest.toml")) + config.options = { + "qemu": { + "img_bin": "qemu-img", + "guests": {"g": {"path": str(tmp_path / "guest")}}, + } + } + cijoe = Cijoe(config, tmp_path / "output", False) + cijoe.set_output_ident("test") + + commands = [] + + def record(cmd, *args, **kwargs): + commands.append(cmd) + return 0, None + + if monkeypatch is not None: + monkeypatch.setattr(cijoe, "run_local", record) + + return Guest(cijoe, config, "g"), commands + + +def test_copy_without_overlay(tmp_path, monkeypatch): + guest, commands = make_guest(tmp_path, monkeypatch) + + assert guest.initialize(tmp_path / "disk.qcow2", overlay=False) == 0 + assert commands == [f"cp {tmp_path / 'disk.qcow2'} {guest.boot_img}"] + + +def test_overlay_by_default(tmp_path, monkeypatch): + guest, commands = make_guest(tmp_path, monkeypatch) + + assert guest.initialize(tmp_path / "disk.qcow2") == 0 + assert commands == [ + f"qemu-img create -f qcow2 -F qcow2 -b {(tmp_path / 'disk.qcow2').resolve()}" + f" {guest.boot_img}" + ] + + +def test_overlay_backing_path_is_absolute(tmp_path, monkeypatch): + """qemu reads a relative backing path from the overlay's directory, not the cwd""" + + monkeypatch.chdir(tmp_path) + guest, commands = make_guest(tmp_path, monkeypatch) + + assert guest.initialize("disk.qcow2") == 0 + assert f"-b {tmp_path.resolve() / 'disk.qcow2'} " in commands[0] + + +@pytest.mark.skipif(shutil.which("qemu-img") is None, reason="needs qemu-img") +def test_overlay_is_a_qcow2_backed_by_the_disk_image(tmp_path): + disk = tmp_path / "disk.qcow2" + subprocess.run(["qemu-img", "create", "-f", "qcow2", str(disk), "64M"], check=True) + + guest, _ = make_guest(tmp_path) + assert guest.initialize(disk) == 0 + + info = subprocess.run( + ["qemu-img", "info", "--output=json", str(guest.boot_img)], + check=True, + capture_output=True, + text=True, + ).stdout + assert f'"backing-filename": "{disk.resolve()}"' in info + assert '"virtual-size": 67108864' in info From 20c04eb6b744f732ed12d4b253809e7a018bba0b Mon Sep 17 00:00:00 2001 From: "Simon A. F. Lund" Date: Fri, 23 Oct 2026 17:41:51 +0200 Subject: [PATCH 2/2] docs(changelog): add the report options to 0.9.61 The report options change landed before the 0.9.61 section existed, so its entry is added now that the section is there. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Simon A. F. Lund --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a8413e..6737c16d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ multi-GB copy and guests from one image share it. `qemu.guest_initialize` takes `--overlay false` to copy as before; `diskimage_from_cloudimage` does so, since it must write the image it produces. +The report options are now `--report-open` to also open the report in a +browser, which is no longer done by default, `--report-skip` to not write it, +and `--report-only` to write it for an existing output and exit. +`-n/--no-report` and `-p/--produce-report` remain as deprecated spellings of +`--report-skip` and `--report-only`, and `-s/--skip-report` is accepted and +ignored, each with a warning. + ## 0.9.59 Renamed the "workflow" concept to "task" across the codebase, docs, and example