diff --git a/README.md b/README.md index f6183b17..6931d789 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,26 @@ Easily manage and keep up to date a large number of images on an OpenStack environment +## Usage + +By default `openstack-image-manager` (and `osism manage images`, which uses it +in the backend) only shows a **preview** of the images that would be uploaded, +a rough estimate of how long that would take and the command to actually +perform the upload. It does not connect to OpenStack and makes no changes: + +``` +openstack-image-manager +``` + +To actually import the images, add `--upload`: + +``` +openstack-image-manager --upload +``` + +See the [documentation](https://osism.tech/docs/guides/operations-guide/openstack/tools/image-manager/) +for all available options. + ## Upstream checksum fields Image versions using the `latest` pointer must specify where to find the diff --git a/openstack_image_manager/main.py b/openstack_image_manager/main.py index 77c7d7ad..d3a0811e 100644 --- a/openstack_image_manager/main.py +++ b/openstack_image_manager/main.py @@ -6,6 +6,7 @@ import yaml import os import re +import shlex import sys import typer import typing @@ -25,6 +26,17 @@ # timeout in seconds for HTTP requests fetching checksum files REQUESTS_TIMEOUT = 30 +# Assumed Glance web-download throughput range (bytes/s) used *only* to derive +# the vague upload-time estimate shown in the default preview. These are +# deliberately rough: actual time depends on the network and Glance backend. +ESTIMATE_THROUGHPUT_FAST = 30 * 1024 * 1024 # ~30 MB/s -> lower time bound +ESTIMATE_THROUGHPUT_SLOW = 10 * 1024 * 1024 # ~10 MB/s -> upper time bound + +# Link shown in the preview for further documentation and options +DOCS_URL = ( + "https://osism.tech/docs/guides/operations-guide/openstack/tools/image-manager/" +) + class ImageManager: def __init__(self) -> None: @@ -33,6 +45,12 @@ def __init__(self) -> None: def create_cli_args( self, debug: bool = typer.Option(False, "--debug", help="Enable debug logging"), + upload: bool = typer.Option( + False, + "--upload", + help="Actually import the images. Without this flag only a preview " + "of the images that would be uploaded is shown.", + ), dry_run: bool = typer.Option( False, "--dry-run", help="Do not perform any changes" ), @@ -294,6 +312,12 @@ def main(self) -> None: # manage images else: + # Without --upload we only show a preview of what would be uploaded + # and make no connection and no changes to the cloud. + if not self.CONF.upload: + self.show_upload_preview() + return + self.create_connection() images = self.read_image_files() managed_images = self.process_images(images) @@ -323,6 +347,147 @@ def main(self) -> None: "please check the output." ) + def collect_planned_uploads(self) -> list: + """ + Collect the images that would be uploaded, based purely on the local + image definitions (no connection to OpenStack is made). + + Honours the same enable/force/filter rules as a real run, as well as + --latest for images of type multi. + + Returns: + A list of (name, url) tuples, one per image version that would be + imported. ``url`` is the download URL (``mirror_url`` if given, + else ``url``) and may be ``None`` if no URL is defined. + """ + planned = [] + for image in self.read_image_files(): + separator = image.get("separator", " ") + version_map = {str(v["version"]): v for v in image.get("versions", [])} + sorted_versions = natsorted(version_map.keys()) + + if image.get("multi") and self.CONF.latest and sorted_versions: + selected_versions = [sorted_versions[-1]] + else: + selected_versions = sorted_versions + + for version in selected_versions: + if image.get("multi"): + name = f"{image['name']}{separator}({version})" + else: + name = f"{image['name']}{separator}{version}" + entry = version_map[version] + url = entry.get("mirror_url", entry.get("url")) + planned.append((name, url)) + + return planned + + def get_remote_size(self, url: str) -> typing.Union[int, None]: + """ + Determine the size of a download in bytes without fetching it. + + Uses the local filesystem for ``file:`` URLs and an HTTP HEAD request + otherwise. Returns ``None`` if the size cannot be determined. + """ + if not url: + return None + + parsed_url = urllib.parse.urlparse(url) + if parsed_url.scheme == "file": + try: + return os.path.getsize(parsed_url.path) + except OSError: + return None + + try: + response = requests.head( + url, allow_redirects=True, timeout=REQUESTS_TIMEOUT + ) + content_length = response.headers.get("Content-Length") + return int(content_length) if content_length is not None else None + except (requests.RequestException, ValueError): + return None + + @staticmethod + def format_size(num_bytes: int) -> str: + """Render a byte count as a human readable string using binary units.""" + size = float(num_bytes) + for unit in ["B", "KiB", "MiB", "GiB"]: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} TiB" + + def build_upload_command(self) -> str: + """Build the equivalent command to actually perform the upload.""" + parts = ["openstack-image-manager", "--upload"] + if self.CONF.cloud != "openstack": + parts += ["--cloud", self.CONF.cloud] + if self.CONF.images != "etc/images/": + parts += ["--images", self.CONF.images] + if self.CONF.filter: + parts += ["--filter", self.CONF.filter] + if self.CONF.latest: + parts.append("--latest") + return shlex.join(parts) + + def show_upload_preview(self) -> None: + """ + Show a preview of the images that would be uploaded, a rough estimate of + how long the upload would take, the command to actually perform it and a + link to the documentation. + + This is the default behaviour when running without ``--upload``. It does + not connect to OpenStack and makes no changes. + """ + planned = self.collect_planned_uploads() + + if not planned: + typer.echo("No enabled images found that would be uploaded.") + typer.echo(f"\nFor more information and options, see:\n {DOCS_URL}") + return + + typer.echo(f"The following {len(planned)} image(s) would be uploaded:\n") + for name, _ in planned: + typer.echo(f" {name}") + + total_bytes = 0 + unknown = 0 + for _, url in planned: + size = self.get_remote_size(url) + if size is None: + unknown += 1 + else: + total_bytes += size + + typer.echo("") + if total_bytes > 0: + suffix = "" + if unknown: + suffix = f" (size of {unknown} image(s) could not be determined)" + typer.echo(f"Total download size: ~{self.format_size(total_bytes)}{suffix}") + + fast_minutes = round(total_bytes / ESTIMATE_THROUGHPUT_FAST / 60) + slow_minutes = round(total_bytes / ESTIMATE_THROUGHPUT_SLOW / 60) + if slow_minutes < 1: + estimate = "less than a minute" + elif fast_minutes == slow_minutes: + estimate = f"~{slow_minutes} min" + else: + estimate = f"~{max(1, fast_minutes)}-{slow_minutes} min" + typer.echo( + f"Estimated upload time: {estimate} " + "(rough estimate, actual time depends on network and Glance backend)" + ) + else: + typer.echo("Total download size could not be determined.") + + typer.echo( + "\nNo changes have been made. To actually upload these images, run:\n" + ) + typer.echo(f" {self.build_upload_command()}") + typer.echo(f"\nFor more information and options, see:\n {DOCS_URL}") + def process_images(self, images) -> set: """Process each image from images.yaml""" diff --git a/playbooks/integration-test.yml b/playbooks/integration-test.yml index 771f5a33..e8d75450 100644 --- a/playbooks/integration-test.yml +++ b/playbooks/integration-test.yml @@ -8,4 +8,4 @@ roles: - role: tox vars: - tox_extra_args: -- --filter 'Cirros' + tox_extra_args: -- --upload --filter 'Cirros' diff --git a/test/unit/test_manage.py b/test/unit/test_manage.py index 338313cf..ed448fa1 100644 --- a/test/unit/test_manage.py +++ b/test/unit/test_manage.py @@ -175,6 +175,7 @@ def setUp(self): latest=True, check_age=False, max_age=90, + upload=False, dry_run=False, use_os_hidden=False, delete=False, @@ -687,6 +688,7 @@ def test_main( mock_unshare_image, ): """test main.ImageManager.main()""" + self.sot.CONF.upload = True mock_read_image_files.return_value = [self.fake_image_dict] mock_process_images.return_value = set() @@ -964,6 +966,7 @@ def test_main_skips_cleanup_on_error( ): """test that main.ImageManager.main() does not clean up outdated images when errors occurred during image processing""" + self.sot.CONF.upload = True mock_read_image_files.return_value = [self.fake_image_dict] def fail_processing(images): @@ -1004,3 +1007,140 @@ def test_schema_url_fields_reject_ftp(self): else: with self.assertRaises(YamaleError): yamale.validate(schema, data) + + @mock.patch("openstack_image_manager.main.ImageManager.read_image_files") + def test_collect_planned_uploads(self, mock_read_image_files): + multi_image = { + "name": "Ubuntu 20.04", + "multi": True, + "versions": [ + {"version": "1", "url": "https://url.com/v1.qcow2"}, + {"version": "2", "url": "https://url.com/v2.qcow2"}, + ], + } + single_image = { + "name": "Cirros", + "multi": False, + "versions": [{"version": "0.6.2", "url": "https://url.com/cirros.img"}], + } + mock_read_image_files.return_value = [multi_image, single_image] + + # without --latest all versions of a multi image are planned + self.sot.CONF.latest = False + planned = self.sot.collect_planned_uploads() + self.assertEqual( + planned, + [ + ("Ubuntu 20.04 (1)", "https://url.com/v1.qcow2"), + ("Ubuntu 20.04 (2)", "https://url.com/v2.qcow2"), + ("Cirros 0.6.2", "https://url.com/cirros.img"), + ], + ) + + # with --latest only the latest version of a multi image is planned + self.sot.CONF.latest = True + planned = self.sot.collect_planned_uploads() + self.assertEqual( + planned, + [ + ("Ubuntu 20.04 (2)", "https://url.com/v2.qcow2"), + ("Cirros 0.6.2", "https://url.com/cirros.img"), + ], + ) + + @mock.patch("openstack_image_manager.main.ImageManager.read_image_files") + def test_collect_planned_uploads_prefers_mirror_url(self, mock_read_image_files): + mock_read_image_files.return_value = [ + { + "name": "Cirros", + "multi": False, + "versions": [ + { + "version": "1", + "url": "https://url.com/original.img", + "mirror_url": "https://mirror.com/mirror.img", + } + ], + } + ] + planned = self.sot.collect_planned_uploads() + self.assertEqual(planned, [("Cirros 1", "https://mirror.com/mirror.img")]) + + @mock.patch("openstack_image_manager.main.requests.head") + def test_get_remote_size(self, mock_head): + mock_head.return_value.headers = {"Content-Length": "2048"} + self.assertEqual(self.sot.get_remote_size("https://url.com/image.img"), 2048) + + # missing Content-Length header -> None + mock_head.return_value.headers = {} + self.assertIsNone(self.sot.get_remote_size("https://url.com/image.img")) + + # request failure -> None + mock_head.side_effect = requests.RequestException() + self.assertIsNone(self.sot.get_remote_size("https://url.com/image.img")) + + # no url at all -> None + self.assertIsNone(self.sot.get_remote_size(None)) + + def test_format_size(self): + self.assertEqual(self.sot.format_size(512), "512.0 B") + self.assertEqual(self.sot.format_size(1024), "1.0 KiB") + self.assertEqual(self.sot.format_size(5 * 1024**3), "5.0 GiB") + + def test_build_upload_command(self): + self.sot.CONF.cloud = "openstack" + self.sot.CONF.images = "etc/images/" + self.sot.CONF.filter = "" + self.sot.CONF.latest = False + self.assertEqual( + self.sot.build_upload_command(), "openstack-image-manager --upload" + ) + + self.sot.CONF.cloud = "mycloud" + self.sot.CONF.filter = "Ubuntu 20.04" + self.sot.CONF.latest = True + self.assertEqual( + self.sot.build_upload_command(), + "openstack-image-manager --upload --cloud mycloud " + "--filter 'Ubuntu 20.04' --latest", + ) + + @mock.patch("openstack_image_manager.main.ImageManager.create_connection") + @mock.patch("openstack_image_manager.main.ImageManager.get_remote_size") + @mock.patch("openstack_image_manager.main.ImageManager.collect_planned_uploads") + @mock.patch("openstack_image_manager.main.typer.echo") + def test_show_upload_preview( + self, mock_echo, mock_collect, mock_size, mock_connection + ): + mock_collect.return_value = [("Cirros 0.6.2", "https://url.com/cirros.img")] + mock_size.return_value = 5 * 1024**3 + + self.sot.CONF.cloud = "openstack" + self.sot.CONF.images = "etc/images/" + self.sot.CONF.filter = "" + self.sot.CONF.latest = False + + self.sot.show_upload_preview() + + # the preview must never connect to OpenStack + mock_connection.assert_not_called() + + output = "\n".join(str(call.args[0]) for call in mock_echo.call_args_list) + self.assertIn("Cirros 0.6.2", output) + self.assertIn("Total download size", output) + self.assertIn("Estimated upload time", output) + self.assertIn("openstack-image-manager --upload", output) + self.assertIn(main.DOCS_URL, output) + + @mock.patch("openstack_image_manager.main.ImageManager.create_connection") + @mock.patch("openstack_image_manager.main.ImageManager.collect_planned_uploads") + @mock.patch("openstack_image_manager.main.typer.echo") + def test_show_upload_preview_no_images( + self, mock_echo, mock_collect, mock_connection + ): + mock_collect.return_value = [] + self.sot.show_upload_preview() + mock_connection.assert_not_called() + output = "\n".join(str(call.args[0]) for call in mock_echo.call_args_list) + self.assertIn("No enabled images found", output) + self.assertIn(main.DOCS_URL, output)