From 4c863630ec9e41d785467cadd669854bdbe3d8da Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 21 Jun 2026 12:54:08 +0200 Subject: [PATCH 1/2] kolla: sync image definitions with kolla-ansible Reconcile all/002-images-kolla.yml with the upstream openstack/kolla-ansible role defaults (ansible/roles/*/defaults/main.yml). Add the image and tag defaults for neutron-ovn-agent, neutron-rpc-server, neutron-periodic-worker, neutron-ovn-maintenance-worker, the keystone httpd image, ironic-pxe-filter and the valkey role (valkey-server, valkey-sentinel). neutron-rpc-server, neutron-periodic-worker and neutron-ovn-maintenance-worker reuse the neutron-server image upstream (no dedicated image is built for them), so they point at neutron_server_image. Add contrib/compare-kolla-images.py to compare the image parameters in all/002-images-kolla.yml against the kolla-ansible role defaults (reports missing, obsolete, aliased and differing image variables). Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- all/002-images-kolla.yml | 20 ++ contrib/compare-kolla-images.py | 345 ++++++++++++++++++++++++++++++++ 2 files changed, 365 insertions(+) create mode 100755 contrib/compare-kolla-images.py diff --git a/all/002-images-kolla.yml b/all/002-images-kolla.yml index cdf7370..61558c2 100644 --- a/all/002-images-kolla.yml +++ b/all/002-images-kolla.yml @@ -68,6 +68,14 @@ redis_tag: "{{ kolla_redis_version|default(kolla_image_version) }}" redis_sentinel_image: "{{ docker_image_url }}redis-sentinel" redis_sentinel_tag: "{{ redis_tag }}" +############################## +# role: valkey + +valkey_image: "{{ docker_image_url }}valkey-server" +valkey_tag: "{{ kolla_valkey_version|default(kolla_image_version) }}" +valkey_sentinel_image: "{{ docker_image_url }}valkey-sentinel" +valkey_sentinel_tag: "{{ valkey_tag }}" + ############################## # role: octavia-certificates @@ -182,6 +190,8 @@ keystone_fernet_image: "{{ docker_image_url }}keystone-fernet" keystone_fernet_tag: "{{ keystone_tag }}" keystone_ssh_image: "{{ docker_image_url }}keystone-ssh" keystone_ssh_tag: "{{ keystone_tag }}" +keystone_httpd_image: "{{ docker_image_url }}httpd" +keystone_httpd_tag: "{{ keystone_tag }}" ############################## # role: vitrage @@ -654,10 +664,18 @@ neutron_ovn_metadata_agent_image: "{{ docker_image_url }}neutron-metadata-agent" neutron_ovn_metadata_agent_tag: "{{ neutron_tag }}" neutron_ovn_vpn_agent_image: "{{ docker_image_url }}neutron-ovn-vpn-agent" neutron_ovn_vpn_agent_tag: "{{ neutron_tag }}" +neutron_ovn_agent_image: "{{ docker_image_url }}neutron-ovn-agent" +neutron_ovn_agent_tag: "{{ neutron_tag }}" neutron_openvswitch_agent_image: "{{ docker_image_url }}neutron-openvswitch-agent" neutron_openvswitch_agent_tag: "{{ neutron_tag }}" neutron_server_image: "{{ docker_image_url }}neutron-server" neutron_server_tag: "{{ neutron_tag }}" +neutron_rpc_server_image: "{{ neutron_server_image }}" +neutron_rpc_server_tag: "{{ neutron_tag }}" +neutron_periodic_worker_image: "{{ neutron_server_image }}" +neutron_periodic_worker_tag: "{{ neutron_tag }}" +neutron_ovn_maintenance_worker_image: "{{ neutron_server_image }}" +neutron_ovn_maintenance_worker_tag: "{{ neutron_tag }}" neutron_bgp_dragent_image: "{{ docker_image_url }}neutron-bgp-dragent" neutron_bgp_dragent_tag: "{{ neutron_tag }}" neutron_infoblox_ipam_agent_image: "{{ docker_image_url }}neutron-infoblox-ipam-agent" @@ -679,6 +697,8 @@ ironic_conductor_image: "{{ docker_image_url }}ironic-conductor" ironic_conductor_tag: "{{ ironic_tag }}" ironic_pxe_image: "{{ docker_image_url }}ironic-pxe" ironic_pxe_tag: "{{ ironic_tag }}" +ironic_pxe_filter_image: "{{ docker_image_url }}ironic-pxe-filter" +ironic_pxe_filter_tag: "{{ ironic_tag }}" ironic_inspector_image: "{{ docker_image_url }}ironic-inspector" ironic_inspector_tag: "{{ kolla_ironic_inspector_version|default(kolla_image_version) }}" ironic_dnsmasq_image: "{{ docker_image_url }}dnsmasq" diff --git a/contrib/compare-kolla-images.py b/contrib/compare-kolla-images.py new file mode 100755 index 0000000..11479f0 --- /dev/null +++ b/contrib/compare-kolla-images.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Compare all/002-images-kolla.yml with the upstream openstack/kolla-ansible. + +Background +---------- +OSISM defines the container images for the OpenStack services in +``all/002-images-kolla.yml``. The canonical source of these image parameters in +kolla-ansible are the role defaults: + + kolla-ansible/ansible/roles//defaults/main.yml + +Every image is defined there with a line of the form:: + + _image: "{{ docker_image_url }}" + +(some images alias another image, e.g. +``neutron_rpc_server_image: "{{ neutron_server_image }}"``). + +This script collects every ``*_image`` variable from those role defaults and +compares the set with the ``*_image`` variables defined in +``all/002-images-kolla.yml``. It reports: + +* MISSING - images defined upstream but not (yet) in OSISM. For these a + ready-to-paste OSISM snippet is generated (aliases are kept as + aliases, see below). +* ALIAS - images that upstream defines as an alias to another image, e.g. + ``neutron_periodic_worker_image: "{{ neutron_server_image }}"``. + No dedicated image is built for these; the service reuses another + image. OSISM must follow the same alias - if it points somewhere + else (usually an invented "{{ docker_image_url }}" that is + not built anywhere) the deployment fails to pull the image. +* OBSOLETE - images defined in OSISM but no longer in upstream. These are + usually services that kolla-ansible removed but OSISM still ships + (monasca, sahara, senlin, swift, ...). Informational only. +* DIFFERS - non-alias images present on both sides whose resolved image name + differs. Some are intentional (OSISM builds its own images, e.g. + neutron-eswitchd); review case by case. + +The exit code is non-zero when images are MISSING or have an ALIAS mismatch, +so the script can be used as a CI guard. + +Usage +----- + contrib/compare-kolla-images.py [--kolla-ansible PATH] [--show-obsolete] + [--show-differs] [--strict] +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from pathlib import Path + +# _image: "" (ignore *_image_full, those are derived downstream) +IMAGE_RE = re.compile(r'^(?P[a-z0-9_]+_image):\s*["\']?(?P.+?)["\']?\s*$') +# _tag: ... and _tag: ... +TAG_RE = re.compile(r"^(?P[a-z0-9_]+_tag):") +# "{{ docker_image_url }}" +DOCKER_URL_RE = re.compile( + r"\{\{\s*docker_image_url\s*\}\}\s*(?P[A-Za-z0-9._-]+)" +) +# "{{ some_other_image }}" (an alias to another image variable) +ALIAS_RE = re.compile(r"^\{\{\s*(?P[a-z0-9_]+_image)\s*\}\}$") + + +def find_kolla_ansible(explicit: str | None) -> Path: + """Locate a kolla-ansible checkout.""" + candidates = [] + if explicit: + candidates.append(Path(explicit).expanduser()) + env = os.environ.get("KOLLA_ANSIBLE_PATH") + if env: + candidates.append(Path(env).expanduser()) + repo_root = Path(__file__).resolve().parent.parent + candidates += [ + repo_root.parent / "kolla-ansible", + repo_root.parent.parent / "kolla-ansible", + Path("~/Repositories/kolla-ansible").expanduser(), + ] + for c in candidates: + if (c / "ansible" / "roles").is_dir(): + return c + tried = "\n ".join(str(c) for c in candidates) + sys.exit( + "error: could not find a kolla-ansible checkout. Use --kolla-ansible " + f"or set KOLLA_ANSIBLE_PATH.\nTried:\n {tried}" + ) + + +def parse_image_vars(path: Path) -> dict[str, str]: + """Return {var_name: raw_value} for every ``*_image`` line in *path*.""" + result: dict[str, str] = {} + for line in path.read_text().splitlines(): + if line.lstrip().startswith("#"): + continue + m = IMAGE_RE.match(line) + if m and not m.group("var").endswith("_image_full"): + result[m.group("var")] = m.group("value").strip() + return result + + +def parse_tag_vars(path: Path) -> set[str]: + """Return the set of ``*_tag`` variable names defined in *path*.""" + tags: set[str] = set() + for line in path.read_text().splitlines(): + if line.lstrip().startswith("#"): + continue + m = TAG_RE.match(line) + if m: + tags.add(m.group("var")) + return tags + + +def collect_kolla_images(ka_root: Path) -> tuple[dict[str, str], dict[str, str]]: + """Collect image variables from all kolla-ansible role defaults. + + Returns (var -> role, var -> raw_value). + """ + var_role: dict[str, str] = {} + var_value: dict[str, str] = {} + roles_dir = ka_root / "ansible" / "roles" + for defaults in sorted(roles_dir.glob("*/defaults/main.yml")): + role = defaults.parent.parent.name + for var, value in parse_image_vars(defaults).items(): + var_role[var] = role + var_value[var] = value + return var_role, var_value + + +def alias_target(value: str) -> str | None: + """Return the referenced variable if *value* is a bare alias. + + e.g. ``"{{ neutron_server_image }}"`` -> ``"neutron_server_image"``; + returns None for ``"{{ docker_image_url }}neutron-server"`` and anything + that is not a plain alias to another ``*_image`` variable. + """ + m = ALIAS_RE.match(value.strip()) + return m.group("ref") if m else None + + +def resolve_image_name(var: str, values: dict[str, str], _seen=None) -> str | None: + """Resolve the image name for *var*, following alias chains. + + Returns the image name (e.g. "neutron-server"), the literal value for + unrecognised patterns, or None. + """ + _seen = _seen or set() + if var in _seen or var not in values: + return None + _seen.add(var) + value = values[var] + m = DOCKER_URL_RE.search(value) + if m: + return m.group("name") + m = ALIAS_RE.match(value) + if m: + return resolve_image_name(m.group("ref"), values, _seen) + return value # unknown pattern, surface it verbatim + + +def suggest_snippet( + missing: list[str], + ka_role: dict[str, str], + ka_value: dict[str, str], + osism_tags: set[str], +) -> str: + """Build a ready-to-paste OSISM snippet for the missing image variables.""" + by_role: dict[str, list[str]] = {} + for var in missing: + by_role.setdefault(ka_role.get(var, "unknown"), []).append(var) + + lines: list[str] = [] + for role in sorted(by_role): + lines.append("##############################") + lines.append(f"# role: {role}") + role_tag = f"{role}_tag" + if role_tag not in osism_tags: + lines.append( + f"# NOTE: '{role_tag}' is not defined yet - add a role tag, e.g.:" + ) + lines.append( + f"# {role_tag}: " + f'"{{{{ kolla_{role}_version|default(kolla_image_version) }}}}"' + ) + lines.append("") + for var in sorted(by_role[role]): + tag_var = var[: -len("_image")] + "_tag" + target = alias_target(ka_value[var]) + if target: + # Upstream reuses another image (no dedicated image is built), + # so keep the alias instead of inventing a non-existent name. + lines.append(f'{var}: "{{{{ {target} }}}}"') + else: + name = resolve_image_name(var, ka_value) or "FIXME" + lines.append(f'{var}: "{{{{ docker_image_url }}}}{name}"') + # Skip a self-referential tag when the service equals the role + # (e.g. valkey_image -> valkey_tag is the role tag itself). + if tag_var != role_tag: + lines.append(f'{tag_var}: "{{{{ {role_tag} }}}}"') + lines.append("") + return "\n".join(lines).rstrip() + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compare all/002-images-kolla.yml with openstack/kolla-ansible.", + ) + parser.add_argument( + "--kolla-ansible", + metavar="PATH", + help="Path to a kolla-ansible checkout " + "(default: autodetect, or $KOLLA_ANSIBLE_PATH).", + ) + parser.add_argument( + "--images-file", + metavar="PATH", + help="Path to the OSISM images file " + "(default: all/002-images-kolla.yml next to this script's repo).", + ) + parser.add_argument( + "--show-obsolete", + action="store_true", + help="List image variables present in OSISM but not upstream.", + ) + parser.add_argument( + "--show-differs", + action="store_true", + help="List image variables whose resolved image name differs.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero on obsolete or differing images as well.", + ) + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parent.parent + images_file = ( + Path(args.images_file).expanduser() + if args.images_file + else repo_root / "all" / "002-images-kolla.yml" + ) + if not images_file.is_file(): + sys.exit(f"error: images file not found: {images_file}") + + ka_root = find_kolla_ansible(args.kolla_ansible) + + ka_role, ka_value = collect_kolla_images(ka_root) + osism_value = parse_image_vars(images_file) + osism_tags = parse_tag_vars(images_file) + + ka_vars = set(ka_value) + osism_vars = set(osism_value) + + missing = sorted(ka_vars - osism_vars) + obsolete = sorted(osism_vars - ka_vars) + + differs: list[tuple[str, str, str]] = [] + alias_mismatch: list[tuple[str, str, str]] = [] + for var in sorted(ka_vars & osism_vars): + up_target = alias_target(ka_value[var]) + if up_target: + # Upstream reuses another image; OSISM must follow the same alias, + # otherwise it references an image that is not built anywhere. + if alias_target(osism_value[var]) != up_target: + alias_mismatch.append((var, up_target, osism_value[var])) + continue + ka_name = resolve_image_name(var, ka_value) + osism_name = resolve_image_name(var, osism_value) + if ka_name != osism_name: + differs.append((var, ka_name or "?", osism_name or "?")) + + print(f"kolla-ansible : {ka_root}") + print(f"images file : {images_file}") + print( + f"upstream images: {len(ka_vars)} " + f"osism images: {len(osism_vars)} " + f"missing: {len(missing)} " + f"alias-mismatch: {len(alias_mismatch)} " + f"obsolete: {len(obsolete)} " + f"differs: {len(differs)}" + ) + print() + + if missing: + print( + f"== MISSING in OSISM ({len(missing)}) " + "- defined upstream, absent in OSISM ==" + ) + for var in missing: + target = alias_target(ka_value[var]) + detail = ( + f"alias -> {{{{ {target} }}}}" + if target + else (resolve_image_name(var, ka_value) or "?") + ) + print(f" {var} -> {detail} [role: {ka_role.get(var, '?')}]") + print() + print("-- suggested snippet for all/002-images-kolla.yml --") + print(suggest_snippet(missing, ka_role, ka_value, osism_tags)) + print() + else: + print("== No missing images. OSISM covers every upstream image. ==\n") + + if alias_mismatch: + print( + f"== ALIAS MISMATCH ({len(alias_mismatch)}) " + "- upstream reuses another image, OSISM does not (likely broken) ==" + ) + for var, target, osism_val in alias_mismatch: + print(f" {var}") + print(f" upstream: {{{{ {target} }}}}") + print(f" osism : {osism_val}") + print(f' fix : {var}: "{{{{ {target} }}}}"') + print() + + if args.show_obsolete and obsolete: + print( + f"== OBSOLETE in OSISM ({len(obsolete)}) " + "- in OSISM, not upstream (often intentional) ==" + ) + for var in obsolete: + print(f" {var}") + print() + + if args.show_differs and differs: + print( + f"== DIFFERS ({len(differs)}) " + "- same variable, different image name (review case by case) ==" + ) + for var, ka_name, osism_name in differs: + print(f" {var}: upstream={ka_name} osism={osism_name}") + print() + + rc = 1 if (missing or alias_mismatch) else 0 + if args.strict and (obsolete or differs): + rc = 1 + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) From 5ab4a2f8e052241bc1f9376e4889512746625432 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Mon, 22 Jun 2026 20:05:10 +0200 Subject: [PATCH 2/2] kolla: clone kolla-ansible for image comparison Clone the kolla-ansible repository to compare against instead of probing for a local checkout: add --repo (default openstack/kolla-ansible) and --branch (default master), shallow-clone into a temporary directory and remove it afterwards. Describe the script as a manual helper rather than a CI gate, and report unresolved "{{ ... }}" templates (e.g. the conditional mariadb and prometheus_server images) as unresolved instead of surfacing the raw template string. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- contrib/compare-kolla-images.py | 120 ++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/contrib/compare-kolla-images.py b/contrib/compare-kolla-images.py index 11479f0..dc382d6 100755 --- a/contrib/compare-kolla-images.py +++ b/contrib/compare-kolla-images.py @@ -37,23 +37,39 @@ differs. Some are intentional (OSISM builds its own images, e.g. neutron-eswitchd); review case by case. -The exit code is non-zero when images are MISSING or have an ALIAS mismatch, -so the script can be used as a CI guard. +This is a manual helper, not a CI gate. ``all/002-images-kolla.yml`` is a +single file covering *all* supported releases, while this script compares it +against *one* kolla-ansible checkout. A superset-vs-single-branch comparison +always produces some noise - MISSING when the checked-out branch is newer than +the file's coverage, OBSOLETE for variables belonging to other releases - so +the exit code cannot cleanly mean "out of sync". It is non-zero when images are +MISSING or have an ALIAS mismatch, which is convenient for a local run, but do +not wire it into CI expecting a reliable signal. Usage ----- - contrib/compare-kolla-images.py [--kolla-ansible PATH] [--show-obsolete] - [--show-differs] [--strict] + contrib/compare-kolla-images.py [--repo REPO] [--branch BRANCH] + [--show-obsolete] [--show-differs] [--strict] + +The kolla-ansible repository is shallow-cloned into a temporary directory for +the duration of the run and removed afterwards (default repo: +openstack/kolla-ansible, default branch: master). """ from __future__ import annotations import argparse -import os +import contextlib import re +import shutil +import subprocess import sys +import tempfile from pathlib import Path +DEFAULT_REPO = "openstack/kolla-ansible" +DEFAULT_BRANCH = "master" + # _image: "" (ignore *_image_full, those are derived downstream) IMAGE_RE = re.compile(r'^(?P[a-z0-9_]+_image):\s*["\']?(?P.+?)["\']?\s*$') # _tag: ... and _tag: ... @@ -66,28 +82,44 @@ ALIAS_RE = re.compile(r"^\{\{\s*(?P[a-z0-9_]+_image)\s*\}\}$") -def find_kolla_ansible(explicit: str | None) -> Path: - """Locate a kolla-ansible checkout.""" - candidates = [] - if explicit: - candidates.append(Path(explicit).expanduser()) - env = os.environ.get("KOLLA_ANSIBLE_PATH") - if env: - candidates.append(Path(env).expanduser()) - repo_root = Path(__file__).resolve().parent.parent - candidates += [ - repo_root.parent / "kolla-ansible", - repo_root.parent.parent / "kolla-ansible", - Path("~/Repositories/kolla-ansible").expanduser(), - ] - for c in candidates: - if (c / "ansible" / "roles").is_dir(): - return c - tried = "\n ".join(str(c) for c in candidates) - sys.exit( - "error: could not find a kolla-ansible checkout. Use --kolla-ansible " - f"or set KOLLA_ANSIBLE_PATH.\nTried:\n {tried}" - ) +def repo_url(repo: str) -> str: + """Expand an ``owner/name`` shorthand to a GitHub URL; pass URLs through.""" + if re.fullmatch(r"[\w.-]+/[\w.-]+", repo): + return f"https://github.com/{repo}.git" + return repo + + +@contextlib.contextmanager +def kolla_ansible_checkout(repo: str, branch: str): + """Shallow-clone *repo* at *branch* into a temp dir, then remove it. + + *repo* is an ``owner/name`` shorthand (cloned from GitHub) or a full git + URL. Yields the path to the checkout for the duration of the ``with`` block. + """ + url = repo_url(repo) + tmp = Path(tempfile.mkdtemp(prefix="kolla-ansible-")) + try: + print(f"cloning {url} ({branch}) ...", file=sys.stderr) + try: + subprocess.run( + ["git", "clone", "--depth", "1", "--branch", branch, url, str(tmp)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except FileNotFoundError: + sys.exit("error: 'git' is required but was not found on PATH.") + except subprocess.CalledProcessError as exc: + sys.exit(f"error: failed to clone {url} ({branch}):\n{exc.stderr.strip()}") + if not (tmp / "ansible" / "roles").is_dir(): + sys.exit( + f"error: {url} ({branch}) does not look like a kolla-ansible " + "checkout (no ansible/roles directory)." + ) + yield tmp + finally: + shutil.rmtree(tmp, ignore_errors=True) def parse_image_vars(path: Path) -> dict[str, str]: @@ -145,7 +177,9 @@ def resolve_image_name(var: str, values: dict[str, str], _seen=None) -> str | No """Resolve the image name for *var*, following alias chains. Returns the image name (e.g. "neutron-server"), the literal value for - unrecognised patterns, or None. + unrecognised plain patterns, or None when the value is an unresolved + template (e.g. a conditional name built from a second ``{{ ... }}``) or + the variable is unknown. """ _seen = _seen or set() if var in _seen or var not in values: @@ -158,7 +192,13 @@ def resolve_image_name(var: str, values: dict[str, str], _seen=None) -> str | No m = ALIAS_RE.match(value) if m: return resolve_image_name(m.group("ref"), values, _seen) - return value # unknown pattern, surface it verbatim + if "{{" in value: + # A template expression we could not resolve to a plain name, e.g. a + # conditional like mariadb_image/prometheus_server_image that inserts a + # second "{{ ... }}". It is not an image name, so report it as + # unresolved instead of surfacing the raw template string. + return None + return value # unknown literal pattern, surface it verbatim def suggest_snippet( @@ -209,10 +249,17 @@ def main() -> int: description="Compare all/002-images-kolla.yml with openstack/kolla-ansible.", ) parser.add_argument( - "--kolla-ansible", - metavar="PATH", - help="Path to a kolla-ansible checkout " - "(default: autodetect, or $KOLLA_ANSIBLE_PATH).", + "--repo", + metavar="REPO", + default=DEFAULT_REPO, + help="kolla-ansible repository to clone, as 'owner/name' (GitHub) or a " + f"full git URL (default: {DEFAULT_REPO}).", + ) + parser.add_argument( + "--branch", + metavar="BRANCH", + default=DEFAULT_BRANCH, + help=f"Branch to clone and compare against (default: {DEFAULT_BRANCH}).", ) parser.add_argument( "--images-file", @@ -246,9 +293,10 @@ def main() -> int: if not images_file.is_file(): sys.exit(f"error: images file not found: {images_file}") - ka_root = find_kolla_ansible(args.kolla_ansible) + ka_source = f"{repo_url(args.repo)} ({args.branch})" + with kolla_ansible_checkout(args.repo, args.branch) as ka_root: + ka_role, ka_value = collect_kolla_images(ka_root) - ka_role, ka_value = collect_kolla_images(ka_root) osism_value = parse_image_vars(images_file) osism_tags = parse_tag_vars(images_file) @@ -273,7 +321,7 @@ def main() -> int: if ka_name != osism_name: differs.append((var, ka_name or "?", osism_name or "?")) - print(f"kolla-ansible : {ka_root}") + print(f"kolla-ansible : {ka_source}") print(f"images file : {images_file}") print( f"upstream images: {len(ka_vars)} "