Preview images by default, require --upload

README.md

@@ -6,6 +6,26 @@
 
 Easily manage and keep up to date a large number of images on an OpenStack environment
 
+## Usage
+
+By default `openstack-image-manager` (and `osism manage images`, which uses it
+in the backend) only shows a **preview** of the images that would be uploaded,
+a rough estimate of how long that would take and the command to actually
+perform the upload. It does not connect to OpenStack and makes no changes:
+
+```
+openstack-image-manager
+```
+
+To actually import the images, add `--upload`:
+
+```
+openstack-image-manager --upload
+```
+
+See the [documentation](https://osism.tech/docs/guides/operations-guide/openstack/tools/image-manager/)
+for all available options.
+
 ## Upstream checksum fields
 
 Image versions using the `latest` pointer must specify where to find the

openstack_image_manager/main.py

@@ -6,6 +6,7 @@
 import yaml
 import os
 import re
+import shlex
 import sys
 import typer
 import typing
@@ -25,6 +26,17 @@
 # timeout in seconds for HTTP requests fetching checksum files
 REQUESTS_TIMEOUT = 30
 
+# Assumed Glance web-download throughput range (bytes/s) used *only* to derive
+# the vague upload-time estimate shown in the default preview. These are
+# deliberately rough: actual time depends on the network and Glance backend.
+ESTIMATE_THROUGHPUT_FAST = 30 * 1024 * 1024  # ~30 MB/s -> lower time bound
+ESTIMATE_THROUGHPUT_SLOW = 10 * 1024 * 1024  # ~10 MB/s -> upper time bound
+
+# Link shown in the preview for further documentation and options
+DOCS_URL = (
+    "https://osism.tech/docs/guides/operations-guide/openstack/tools/image-manager/"
+)
+
 
 class ImageManager:
     def __init__(self) -> None:
@@ -33,6 +45,12 @@ def __init__(self) -> None:
     def create_cli_args(
         self,
         debug: bool = typer.Option(False, "--debug", help="Enable debug logging"),
+        upload: bool = typer.Option(
+            False,
+            "--upload",
+            help="Actually import the images. Without this flag only a preview "
+            "of the images that would be uploaded is shown.",
+        ),
         dry_run: bool = typer.Option(
             False, "--dry-run", help="Do not perform any changes"
         ),
@@ -294,6 +312,12 @@ def main(self) -> None:
 
         # manage images
         else:
+            # Without --upload we only show a preview of what would be uploaded
+            # and make no connection and no changes to the cloud.
+            if not self.CONF.upload:
+                self.show_upload_preview()
+                return
+
             self.create_connection()
             images = self.read_image_files()
             managed_images = self.process_images(images)
@@ -323,6 +347,147 @@ def main(self) -> None:
                 "please check the output."
             )
 
+    def collect_planned_uploads(self) -> list:
+        """
+        Collect the images that would be uploaded, based purely on the local
+        image definitions (no connection to OpenStack is made).
+
+        Honours the same enable/force/filter rules as a real run, as well as
+        --latest for images of type multi.
+
+        Returns:
+            A list of (name, url) tuples, one per image version that would be
+            imported. ``url`` is the download URL (``mirror_url`` if given,
+            else ``url``) and may be ``None`` if no URL is defined.
+        """
+        planned = []
+        for image in self.read_image_files():
+            separator = image.get("separator", " ")
+            version_map = {str(v["version"]): v for v in image.get("versions", [])}
+            sorted_versions = natsorted(version_map.keys())
+
+            if image.get("multi") and self.CONF.latest and sorted_versions:
+                selected_versions = [sorted_versions[-1]]
+            else:
+                selected_versions = sorted_versions
+
+            for version in selected_versions:
+                if image.get("multi"):
+                    name = f"{image['name']}{separator}({version})"
+                else:
+                    name = f"{image['name']}{separator}{version}"
+                entry = version_map[version]
+                url = entry.get("mirror_url", entry.get("url"))
+                planned.append((name, url))
+
+        return planned
+
+    def get_remote_size(self, url: str) -> typing.Union[int, None]:
+        """
+        Determine the size of a download in bytes without fetching it.
+
+        Uses the local filesystem for ``file:`` URLs and an HTTP HEAD request
+        otherwise. Returns ``None`` if the size cannot be determined.
+        """
+        if not url:
+            return None
+
+        parsed_url = urllib.parse.urlparse(url)
+        if parsed_url.scheme == "file":
+            try:
+                return os.path.getsize(parsed_url.path)
+            except OSError:
+                return None
+
+        try:
+            response = requests.head(
+                url, allow_redirects=True, timeout=REQUESTS_TIMEOUT
+            )
+            content_length = response.headers.get("Content-Length")
+            return int(content_length) if content_length is not None else None
+        except (requests.RequestException, ValueError):
+            return None
+
+    @staticmethod
+    def format_size(num_bytes: int) -> str:
+        """Render a byte count as a human readable string using binary units."""
+        size = float(num_bytes)
+        for unit in ["B", "KiB", "MiB", "GiB"]:
+            if size < 1024:
+                return f"{size:.1f} {unit}"
+            size /= 1024
+        return f"{size:.1f} TiB"
+
+    def build_upload_command(self) -> str:
+        """Build the equivalent command to actually perform the upload."""
+        parts = ["openstack-image-manager", "--upload"]
+        if self.CONF.cloud != "openstack":
+            parts += ["--cloud", self.CONF.cloud]
+        if self.CONF.images != "etc/images/":
+            parts += ["--images", self.CONF.images]
+        if self.CONF.filter:
+            parts += ["--filter", self.CONF.filter]
+        if self.CONF.latest:
+            parts.append("--latest")
+        return shlex.join(parts)
+
+    def show_upload_preview(self) -> None:
+        """
+        Show a preview of the images that would be uploaded, a rough estimate of
+        how long the upload would take, the command to actually perform it and a
+        link to the documentation.
+
+        This is the default behaviour when running without ``--upload``. It does
+        not connect to OpenStack and makes no changes.
+        """
+        planned = self.collect_planned_uploads()
+
+        if not planned:
+            typer.echo("No enabled images found that would be uploaded.")
+            typer.echo(f"\nFor more information and options, see:\n  {DOCS_URL}")
+            return
+
+        typer.echo(f"The following {len(planned)} image(s) would be uploaded:\n")
+        for name, _ in planned:
+            typer.echo(f"  {name}")
+
+        total_bytes = 0
+        unknown = 0
+        for _, url in planned:
+            size = self.get_remote_size(url)
+            if size is None:
+                unknown += 1
+            else:
+                total_bytes += size
+
+        typer.echo("")
+        if total_bytes > 0:
+            suffix = ""
+            if unknown:
+                suffix = f" (size of {unknown} image(s) could not be determined)"
+            typer.echo(f"Total download size: ~{self.format_size(total_bytes)}{suffix}")
+
+            fast_minutes = round(total_bytes / ESTIMATE_THROUGHPUT_FAST / 60)
+            slow_minutes = round(total_bytes / ESTIMATE_THROUGHPUT_SLOW / 60)
+            if slow_minutes < 1:
+                estimate = "less than a minute"
+            elif fast_minutes == slow_minutes:
+                estimate = f"~{slow_minutes} min"
+            else:
+                estimate = f"~{max(1, fast_minutes)}-{slow_minutes} min"
+            typer.echo(
+                f"Estimated upload time: {estimate} "
+                "(rough estimate, actual time depends on network and Glance backend)"
+            )
+        else:
+            typer.echo("Total download size could not be determined.")
+
+        typer.echo(
+            "\nNo changes have been made. To actually upload these images, run:\n"
+        )
+        typer.echo(f"  {self.build_upload_command()}")
+        typer.echo(f"\nFor more information and options, see:\n  {DOCS_URL}")
+
     def process_images(self, images) -> set:
         """Process each image from images.yaml"""
 

playbooks/integration-test.yml

@@ -8,4 +8,4 @@
   roles:
     - role: tox
       vars:
-        tox_extra_args: -- --filter 'Cirros'
+        tox_extra_args: -- --upload --filter 'Cirros'

test/unit/test_manage.py

@@ -175,6 +175,7 @@ def setUp(self):
             latest=True,
             check_age=False,
             max_age=90,
+            upload=False,
             dry_run=False,
             use_os_hidden=False,
             delete=False,
@@ -687,6 +688,7 @@ def test_main(
         mock_unshare_image,
     ):
         """test main.ImageManager.main()"""
+        self.sot.CONF.upload = True
         mock_read_image_files.return_value = [self.fake_image_dict]
         mock_process_images.return_value = set()
 
@@ -964,6 +966,7 @@ def test_main_skips_cleanup_on_error(
     ):
         """test that main.ImageManager.main() does not clean up outdated
         images when errors occurred during image processing"""
+        self.sot.CONF.upload = True
         mock_read_image_files.return_value = [self.fake_image_dict]
 
         def fail_processing(images):
@@ -1004,3 +1007,140 @@ def test_schema_url_fields_reject_ftp(self):
                     else:
                         with self.assertRaises(YamaleError):
                             yamale.validate(schema, data)
+
+    @mock.patch("openstack_image_manager.main.ImageManager.read_image_files")
+    def test_collect_planned_uploads(self, mock_read_image_files):
+        multi_image = {
+            "name": "Ubuntu 20.04",
+            "multi": True,
+            "versions": [
+                {"version": "1", "url": "https://url.com/v1.qcow2"},
+                {"version": "2", "url": "https://url.com/v2.qcow2"},
+            ],
+        }
+        single_image = {
+            "name": "Cirros",
+            "multi": False,
+            "versions": [{"version": "0.6.2", "url": "https://url.com/cirros.img"}],
+        }
+        mock_read_image_files.return_value = [multi_image, single_image]
+
+        # without --latest all versions of a multi image are planned
+        self.sot.CONF.latest = False
+        planned = self.sot.collect_planned_uploads()
+        self.assertEqual(
+            planned,
+            [
+                ("Ubuntu 20.04 (1)", "https://url.com/v1.qcow2"),
+                ("Ubuntu 20.04 (2)", "https://url.com/v2.qcow2"),
+                ("Cirros 0.6.2", "https://url.com/cirros.img"),
+            ],
+        )
+
+        # with --latest only the latest version of a multi image is planned
+        self.sot.CONF.latest = True
+        planned = self.sot.collect_planned_uploads()
+        self.assertEqual(
+            planned,
+            [
+                ("Ubuntu 20.04 (2)", "https://url.com/v2.qcow2"),
+                ("Cirros 0.6.2", "https://url.com/cirros.img"),
+            ],
+        )
+
+    @mock.patch("openstack_image_manager.main.ImageManager.read_image_files")
+    def test_collect_planned_uploads_prefers_mirror_url(self, mock_read_image_files):
+        mock_read_image_files.return_value = [
+            {
+                "name": "Cirros",
+                "multi": False,
+                "versions": [
+                    {
+                        "version": "1",
+                        "url": "https://url.com/original.img",
+                        "mirror_url": "https://mirror.com/mirror.img",
+                    }
+                ],
+            }
+        ]
+        planned = self.sot.collect_planned_uploads()
+        self.assertEqual(planned, [("Cirros 1", "https://mirror.com/mirror.img")])
+
+    @mock.patch("openstack_image_manager.main.requests.head")
+    def test_get_remote_size(self, mock_head):
+        mock_head.return_value.headers = {"Content-Length": "2048"}
+        self.assertEqual(self.sot.get_remote_size("https://url.com/image.img"), 2048)
+
+        # missing Content-Length header -> None
+        mock_head.return_value.headers = {}
+        self.assertIsNone(self.sot.get_remote_size("https://url.com/image.img"))
+
+        # request failure -> None
+        mock_head.side_effect = requests.RequestException()
+        self.assertIsNone(self.sot.get_remote_size("https://url.com/image.img"))
+
+        # no url at all -> None
+        self.assertIsNone(self.sot.get_remote_size(None))
+
+    def test_format_size(self):
+        self.assertEqual(self.sot.format_size(512), "512.0 B")
+        self.assertEqual(self.sot.format_size(1024), "1.0 KiB")
+        self.assertEqual(self.sot.format_size(5 * 1024**3), "5.0 GiB")
+
+    def test_build_upload_command(self):
+        self.sot.CONF.cloud = "openstack"
+        self.sot.CONF.images = "etc/images/"
+        self.sot.CONF.filter = ""
+        self.sot.CONF.latest = False
+        self.assertEqual(
+            self.sot.build_upload_command(), "openstack-image-manager --upload"
+        )
+
+        self.sot.CONF.cloud = "mycloud"
+        self.sot.CONF.filter = "Ubuntu 20.04"
+        self.sot.CONF.latest = True
+        self.assertEqual(
+            self.sot.build_upload_command(),
+            "openstack-image-manager --upload --cloud mycloud "
+            "--filter 'Ubuntu 20.04' --latest",
+        )
+
+    @mock.patch("openstack_image_manager.main.ImageManager.create_connection")
+    @mock.patch("openstack_image_manager.main.ImageManager.get_remote_size")
+    @mock.patch("openstack_image_manager.main.ImageManager.collect_planned_uploads")
+    @mock.patch("openstack_image_manager.main.typer.echo")
+    def test_show_upload_preview(
+        self, mock_echo, mock_collect, mock_size, mock_connection
+    ):
+        mock_collect.return_value = [("Cirros 0.6.2", "https://url.com/cirros.img")]
+        mock_size.return_value = 5 * 1024**3
+
+        self.sot.CONF.cloud = "openstack"
+        self.sot.CONF.images = "etc/images/"
+        self.sot.CONF.filter = ""
+        self.sot.CONF.latest = False
+
+        self.sot.show_upload_preview()
+
+        # the preview must never connect to OpenStack
+        mock_connection.assert_not_called()
+
+        output = "\n".join(str(call.args[0]) for call in mock_echo.call_args_list)
+        self.assertIn("Cirros 0.6.2", output)
+        self.assertIn("Total download size", output)
+        self.assertIn("Estimated upload time", output)
+        self.assertIn("openstack-image-manager --upload", output)
+        self.assertIn(main.DOCS_URL, output)
+
+    @mock.patch("openstack_image_manager.main.ImageManager.create_connection")
+    @mock.patch("openstack_image_manager.main.ImageManager.collect_planned_uploads")
+    @mock.patch("openstack_image_manager.main.typer.echo")
+    def test_show_upload_preview_no_images(
+        self, mock_echo, mock_collect, mock_connection
+    ):
+        mock_collect.return_value = []
+        self.sot.show_upload_preview()
+        mock_connection.assert_not_called()
+        output = "\n".join(str(call.args[0]) for call in mock_echo.call_args_list)
+        self.assertIn("No enabled images found", output)
+        self.assertIn(main.DOCS_URL, output)