From b1ba6939a571a8e22b3611f071db8958bbef706b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 14:00:33 +0000 Subject: [PATCH 1/4] fix: scope registry credentials to the target host and harden auth handling - New auth.py: write a private (0600) docker config that only contains the target registry, so credentials are no longer offered to the source registries. Previously two configs were created for src+dst and the second silently overwrote the first. - Use the https://index.docker.io/v1/ key for Docker Hub. - Pass the auth env to docker/podman/helm too (credentials were ignored there) and authenticate 'helm push' via HELM_REGISTRY_CONFIG. - --ca-cert is appended to the system CA bundle instead of replacing it, and is honoured by helm. - Add --password-stdin, REGISTRY_USERNAME/REGISTRY_PASSWORD and --auth-registry; warn on --password; require username and password together. - Remove duplicated per-tool auth/cleanup blocks in images.py. - README: correct nonroot UID (65532), drop useless /root mounts, document auth. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013XXYXZzMgKN2pShvj9LqVJ --- .github/workflows/schedule-rebuild.yml | 1 - README.md | 19 +- src/auth.py | 82 ++++++++ src/helm.py | 26 ++- src/images.py | 281 +++++-------------------- src/main.py | 48 ++++- 6 files changed, 210 insertions(+), 247 deletions(-) create mode 100644 src/auth.py diff --git a/.github/workflows/schedule-rebuild.yml b/.github/workflows/schedule-rebuild.yml index 3835c46..849b240 100644 --- a/.github/workflows/schedule-rebuild.yml +++ b/.github/workflows/schedule-rebuild.yml @@ -11,7 +11,6 @@ jobs: permissions: contents: read packages: write - # needed for GHCR login id-token: none steps: diff --git a/README.md b/README.md index 72baf08..4bf7297 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,22 @@ helm-mirror \ --dry-run ``` +**Authenticate against the target registry:** + +```bash +echo "$REGISTRY_PASSWORD" | helm-mirror \ + --chart bitnami/nginx \ + --target-registry registry.internal.corp \ + --username ci-bot --password-stdin \ + --ca-cert /etc/ssl/internal-ca.pem +``` + +Credentials can also come from `REGISTRY_USERNAME` / `REGISTRY_PASSWORD`. Avoid `--password`, which is visible in the process list. Credentials are only valid for the host of `--target-registry` (override with `--auth-registry`) and are never sent to the source registries. `--ca-cert` is added on top of the system trust store (not supported for `docker`/`podman`). + **Run via Docker:** ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ ghcr.io/thenomadbeyond/helm-mirror-cli:latest \ --chart bitnami/nginx \ --target-registry registry.internal.corp \ @@ -129,13 +140,12 @@ When running the Docker image, you may need to mount volumes for: - Output directories (for `--save-images`/`--images-dir` and `--save-chart`/`--chart-dir`) - Temporary storage (if the default `/tmp` is not writable or you want to persist temporary files) -The image runs as a nonroot user (UID 1000). Ensure that any mounted directories are writable by this user. +The image runs as a nonroot user (UID 65532). Ensure that any mounted directories are writable by this user. ### Example: Using Local Charts and Saving Output ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ -v $(pwd)/local-charts:/charts:ro \ -v $(pwd)/output:/output:rw \ -v $(pwd)/tmp:/tmp:rw \ @@ -161,7 +171,6 @@ If you prefer not to mount `/tmp`, you can set the `TMPDIR` environment variable ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ -v $(pwd)/local-charts:/charts:ro \ -v $(pwd)/output:/output:rw \ -v $(pwd)/tmp:/tmp:rw \ @@ -177,7 +186,7 @@ docker run --rm \ Note: The image already uses `/tmp` as the default temporary directory, so mounting a host directory to `/tmp` is often sufficient. ### Important Notes -- The nonroot user in the container has UID 1000. If you get permission errors, ensure the host directories are writable by UID 1000 (e.g., `chmod a+rw` or change ownership to 1000:1000). +- The nonroot user in the container has UID 65532. If you get permission errors, ensure the host directories are writable by UID 65532 (e.g., `chmod a+rw` or change ownership to 65532:65532). - For reading local charts, the mount can be read-only (`:ro`). - For writing output or temporary files, the mount must be read-write (`:rw`). ## CI/CD Pipelines diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..e9f0bc5 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,82 @@ +# mirror_tool/auth.py +import contextlib +import json +import os +import shutil +import ssl +import tempfile +from base64 import b64encode + +_DOCKER_HUB = "https://index.docker.io/v1/" +_HUB_HOSTS = {"docker.io", "index.docker.io", "registry-1.docker.io"} +_SYSTEM_CA_FALLBACKS = ( + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", +) + + +def registry_host(target): + """Host[:port] of a target registry string such as host:5000/prefix.""" + return target.split("://", 1)[-1].split("/", 1)[0] + + +def _system_ca_bundle(): + for path in (ssl.get_default_verify_paths().cafile, *_SYSTEM_CA_FALLBACKS): + if path and os.path.isfile(path): + return path + return None + + +def _write_private(path, text): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as f: + f.write(text) + + +@contextlib.contextmanager +def registry_env(tool, registry=None, username=None, password=None, ca_cert=None): + """Yield a subprocess environment carrying registry credentials / a CA. + + Credentials are written to a private temp config that only contains an + entry for `registry`, so they are never offered to any other host (e.g. + the upstream source registry). The temp dir is always removed on exit. + """ + env = os.environ.copy() + tmp = tempfile.mkdtemp(prefix="helm-mirror-") + try: + if password is not None: + if not (username and registry): + raise ValueError("credentials need both a username and a registry") + key = _DOCKER_HUB if registry in _HUB_HOSTS else registry + auth = b64encode(f"{username}:{password}".encode()).decode() + cfg = os.path.join(tmp, "config.json") + _write_private(cfg, json.dumps({"auths": {key: {"auth": auth}}})) + # docker/crane/skopeo read DOCKER_CONFIG, podman/skopeo read + # REGISTRY_AUTH_FILE, helm reads HELM_REGISTRY_CONFIG. + env["DOCKER_CONFIG"] = tmp + env["REGISTRY_AUTH_FILE"] = cfg + env["HELM_REGISTRY_CONFIG"] = cfg + + if ca_cert: + if tool in ("docker", "podman"): + print( + f"[WARN] --ca-cert is not supported for {tool}. " + "Configure the daemon's trust store for custom CAs." + ) + else: + # SSL_CERT_FILE replaces the trust store, so append the custom CA + # to the system bundle instead of losing public registries. + bundle = os.path.join(tmp, "ca-bundle.pem") + parts = [] + system = _system_ca_bundle() + if system: + with open(system) as f: + parts.append(f.read()) + with open(ca_cert) as f: + parts.append(f.read()) + _write_private(bundle, "\n".join(parts)) + env["SSL_CERT_FILE"] = bundle + + yield env + finally: + shutil.rmtree(tmp, ignore_errors=True) diff --git a/src/helm.py b/src/helm.py index a5e5954..56d3e30 100644 --- a/src/helm.py +++ b/src/helm.py @@ -6,16 +6,18 @@ import atexit import os +from auth import registry_env -def run(cmd): - result = subprocess.run(cmd, capture_output=True, text=True) + +def run(cmd, env=None): + result = subprocess.run(cmd, capture_output=True, text=True, env=env) if result.returncode != 0: cmd_str = " ".join(cmd) if isinstance(cmd, list) else cmd raise Exception(f"Command failed: {cmd_str}\n{result.stderr.strip()}") return result.stdout -def pull_chart(chart, version=None, repo_url=None, insecure=False): +def pull_chart(chart, version=None, repo_url=None, insecure=False, ca_cert=None): if os.path.isdir(chart): return chart @@ -33,6 +35,11 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): atexit.register(shutil.rmtree, parent_tmp, True) tmp = os.path.join(parent_tmp, "chart") + with registry_env("helm", ca_cert=ca_cert) as env: + return _pull(chart, version, repo_url, insecure, tmp, env) + + +def _pull(chart, version, repo_url, insecure, tmp, env): repo_added = None if repo_url: @@ -40,8 +47,8 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): add_cmd = ["helm", "repo", "add", repo_name, repo_url] if insecure: add_cmd.append("--insecure-skip-tls-verify") - run(add_cmd) - run(["helm", "repo", "update", repo_name]) + run(add_cmd, env) + run(["helm", "repo", "update", repo_name], env) repo_added = repo_name chart = f"{repo_name}/{chart}" @@ -52,11 +59,11 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): cmd.append("--insecure-skip-tls-verify") try: - run(cmd) + run(cmd, env) finally: if repo_added: subprocess.run( - ["helm", "repo", "remove", repo_added], capture_output=True + ["helm", "repo", "remove", repo_added], capture_output=True, env=env ) dirs = os.listdir(tmp) @@ -76,7 +83,7 @@ def render_chart(chart_path, values=None): return run(cmd) -def push_chart(chart_path, target_registry, chart_target=None, insecure=False): +def push_chart(chart_path, target_registry, chart_target=None, insecure=False, auth=None): pkg_dir = tempfile.mkdtemp() try: run(["helm", "package", chart_path, "--destination", pkg_dir]) @@ -93,7 +100,8 @@ def push_chart(chart_path, target_registry, chart_target=None, insecure=False): cmd = ["helm", "push", tgz, repo] if insecure: cmd.append("--insecure-skip-tls-verify") - run(cmd) + with registry_env("helm", **(auth or {})) as env: + run(cmd, env) print(f"[INFO] Chart pushed: {os.path.basename(tgz)} -> {repo}") finally: shutil.rmtree(pkg_dir, ignore_errors=True) diff --git a/src/images.py b/src/images.py index a1d7a88..a30a896 100644 --- a/src/images.py +++ b/src/images.py @@ -2,12 +2,11 @@ import subprocess import re import os -import tempfile -import base64 -import json import yaml +from auth import registry_env + def _walk_for_images(node, images): """Recursively walk a parsed YAML node and collect image references. @@ -51,75 +50,6 @@ def _walk_for_images(node, images): _walk_for_images(value, images) -def _get_registry_from_image(image): - """Extract registry hostname from image reference.""" - # Handle various image formats: - # - registry/namespace/repo:tag - # - namespace/repo:tag (defaults to docker.io) - # - repo:tag (defaults to docker.io) - # - registry:port/namespace/repo:tag - if '/' not in image: - # No slash, could be just repo or repo:tag - if ':' in image and not image.startswith(('http://', 'https://')): - # Might be repo:tag - return 'docker.io' - else: - # Just repo (unlikely but handle) - return 'docker.io' - - # Split on first '/' to get potential registry - parts = image.split('/', 1) - first_part = parts[0] - - # If first part contains a dot or colon, it's likely a registry - if '.' in first_part or ':' in first_part: - return first_part - else: - # No dot or colon in first part, so it's likely a namespace (e.g., library, bitnami) - # Default to docker.io - return 'docker.io' - - -def _create_docker_auth_config(username, password, registry=None): - """Create a Docker-style auth config for the given credentials. - - Returns a tuple of (temp_dir_path, env_dict) where: - - temp_dir_path: Path to temporary directory containing config.json - - env_dict: Environment variables to set (e.g., {'DOCKER_CONFIG': temp_dir_path}) - """ - if not username and not password: - return None, {} - - # Create temporary directory - temp_dir = tempfile.mkdtemp(prefix='helm-mirror-auth-') - - # Determine registry if not provided - if registry is None: - # We'll need to determine this from context - for now, require registry param - # This function will be called from context where we know the image - registry = 'docker.io' # fallback - - # Create auth string - auth = base64.b64encode(f"{username}:{password}".encode()).decode() - - # Create Docker config structure - config = { - "auths": { - registry: { - "auth": auth - } - } - } - - # Write config.json - config_path = os.path.join(temp_dir, 'config.json') - with open(config_path, 'w') as f: - json.dump(config, f, indent=2) - - # Return temp dir and environment variables - return temp_dir, {"DOCKER_CONFIG": temp_dir} - - def extract_images(rendered_yaml, tools): images = set() @@ -160,46 +90,21 @@ def extract_images(rendered_yaml, tools): return images -def _image_exists(image, tools, insecure=False, ca_cert=None, username=None, password=None): +def _image_exists(image, tools, insecure=False, auth=None): """Return True when the image already exists in the registry.""" - copy_tool = tools.get("copy") - env = os.environ.copy() - - # Handle CA certificate - if ca_cert and copy_tool in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and copy_tool in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {copy_tool}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - auth_temp_dir = None - if username or password: - # Determine registry for auth - registry = _get_registry_from_image(image) - auth_temp_dir, auth_env = _create_docker_auth_config(username, password, registry) - if auth_temp_dir: - env.update(auth_env) - - try: - if copy_tool == "crane": - cmd = ["crane", "manifest", image] - if insecure: - cmd.append("--insecure") - return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 - elif copy_tool == "skopeo": - cmd = ["skopeo", "inspect", f"docker://{image}"] - if insecure: - cmd.append("--tls-verify=false") - return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 + tool = tools.get("copy") + if tool == "crane": + cmd = ["crane", "manifest", image] + if insecure: + cmd.append("--insecure") + elif tool == "skopeo": + cmd = ["skopeo", "inspect", f"docker://{image}"] + if insecure: + cmd.append("--tls-verify=false") + else: return False - finally: - # Clean up temporary auth directory - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + with registry_env(tool, **(auth or {})) as env: + return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 def mirror_images( @@ -210,9 +115,7 @@ def mirror_images( dry_run=False, save_dir=None, insecure=False, - ca_cert=None, - username=None, - password=None, + auth=None, parallel=1, skip_existing=False, ): @@ -221,17 +124,17 @@ def mirror_images( def _process(img): if save_dir is not None: - tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, ca_cert, username, password) + tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, auth) print(f"[INFO] {img} -> {tar_path}") return tar_path new = rewrite_image(img, registry, prefix) - if skip_existing and not dry_run and _image_exists(new, tools, insecure, ca_cert, username, password): + if skip_existing and not dry_run and _image_exists(new, tools, insecure, auth): print(f"[SKIP] {img} -> {new} (already exists)") return new print(f"[INFO] {img} -> {new}") if not dry_run: - copy_image(img, new, tools, insecure, ca_cert, username, password) + copy_image(img, new, tools, insecure, auth) return new if parallel > 1: @@ -288,135 +191,63 @@ def _image_to_filename(image): return f"{safe}.tar" -def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, ca_cert=None, username=None, password=None): +def _warn_docker_insecure(): + print( + "[WARN] --insecure has no effect for docker at the command level. " + "Add the registry to insecure-registries in /etc/docker/daemon.json." + ) + + +def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, auth=None): os.makedirs(output_dir, exist_ok=True) tar_path = os.path.join(output_dir, _image_to_filename(image)) if dry_run: return tar_path - if username or password: - # Authentication is handled via temporary Docker config - pass - - env = os.environ.copy() - auth_temp_dir = None - - # Handle CA certificate - if ca_cert and tools["copy"] in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and tools["copy"] in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {tools['copy']}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - if username or password: - # Determine registry for auth - registry = _get_registry_from_image(image) - auth_temp_dir, auth_env = _create_docker_auth_config(username, password, registry) - if auth_temp_dir: - env.update(auth_env) - - try: - if tools["copy"] == "crane": - cmd = ["crane", "pull"] - if insecure: - cmd.append("--insecure") + tool = tools["copy"] + tls = ["--tls-verify=false"] if insecure else [] + with registry_env(tool, **(auth or {})) as env: + if tool == "crane": + cmd = ["crane", "pull"] + (["--insecure"] if insecure else []) subprocess.run(cmd + ["--format=tarball", image, tar_path], check=True, env=env) - elif tools["copy"] == "skopeo": - src = f"docker://{image}" - dst = f"docker-archive:{tar_path}" - cmd = ["skopeo", "copy"] + elif tool == "skopeo": + cmd = ["skopeo", "copy"] + (["--src-tls-verify=false"] if insecure else []) + subprocess.run(cmd + [f"docker://{image}", f"docker-archive:{tar_path}"], check=True, env=env) + elif tool == "docker": if insecure: - cmd += ["--src-tls-verify=false"] - subprocess.run(cmd + [src, dst], check=True, env=env) - elif tools["copy"] == "docker": - if insecure: - print( - "[WARN] --insecure has no effect for docker at the command level. " - "Add the registry to insecure-registries in /etc/docker/daemon.json." - ) - subprocess.run(["docker", "pull", image], check=True) - subprocess.run(["docker", "save", image, "-o", tar_path], check=True) + _warn_docker_insecure() + subprocess.run(["docker", "pull", image], check=True, env=env) + subprocess.run(["docker", "save", image, "-o", tar_path], check=True, env=env) else: - tls = ["--tls-verify=false"] if insecure else [] - subprocess.run(["podman", "pull"] + tls + [image], check=True) - subprocess.run(["podman", "save", image, "-o", tar_path], check=True) - finally: - # Clean up temporary auth directory - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + subprocess.run(["podman", "pull"] + tls + [image], check=True, env=env) + subprocess.run(["podman", "save", image, "-o", tar_path], check=True, env=env) return tar_path -def copy_image(src, dst, tools, insecure=False, ca_cert=None, username=None, password=None): - if username or password: - # Authentication is handled via temporary Docker config - pass - - env = os.environ.copy() - auth_temp_dirs = [] # Track temp dirs for cleanup - - # Handle CA certificate - if ca_cert and tools["copy"] in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and tools["copy"] in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {tools['copy']}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - need auth for both source and destination - if username or password: - # Create auth for source image - src_registry = _get_registry_from_image(src) - src_auth_temp_dir, src_auth_env = _create_docker_auth_config(username, password, src_registry) - if src_auth_temp_dir: - auth_temp_dirs.append(src_auth_temp_dir) - env.update(src_auth_env) - - # Create auth for destination image (may be same credentials) - dst_registry = _get_registry_from_image(dst) - dst_auth_temp_dir, dst_auth_env = _create_docker_auth_config(username, password, dst_registry) - if dst_auth_temp_dir: - auth_temp_dirs.append(dst_auth_temp_dir) - env.update(dst_auth_env) - - try: - if tools["copy"] == "crane": - cmd = ["crane", "copy"] - if insecure: - cmd.append("--insecure") +def copy_image(src, dst, tools, insecure=False, auth=None): + tool = tools["copy"] + tls = ["--tls-verify=false"] if insecure else [] + with registry_env(tool, **(auth or {})) as env: + if tool == "crane": + cmd = ["crane", "copy"] + (["--insecure"] if insecure else []) subprocess.run(cmd + [src, dst], check=True, env=env) - elif tools["copy"] == "skopeo": + elif tool == "skopeo": cmd = ["skopeo", "copy"] if insecure: cmd += ["--src-tls-verify=false", "--dest-tls-verify=false"] subprocess.run(cmd + [f"docker://{src}", f"docker://{dst}"], check=True, env=env) - elif tools["copy"] == "docker": + elif tool == "docker": if insecure: - print( - "[WARN] --insecure has no effect for docker at the command level. " - "Add the registry to insecure-registries in /etc/docker/daemon.json." - ) - subprocess.run(["docker", "pull", src], check=True) - subprocess.run(["docker", "tag", src, dst], check=True) - subprocess.run(["docker", "push", dst], check=True) + _warn_docker_insecure() + subprocess.run(["docker", "pull", src], check=True, env=env) + subprocess.run(["docker", "tag", src, dst], check=True, env=env) + subprocess.run(["docker", "push", dst], check=True, env=env) else: - tls = ["--tls-verify=false"] if insecure else [] - subprocess.run(["podman", "pull"] + tls + [src], check=True) - subprocess.run(["podman", "tag", src, dst], check=True) - subprocess.run(["podman", "push"] + tls + [dst], check=True) - finally: - # Clean up temporary auth directories - for auth_temp_dir in auth_temp_dirs: - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + subprocess.run(["podman", "pull"] + tls + [src], check=True, env=env) + subprocess.run(["podman", "tag", src, dst], check=True, env=env) + subprocess.run(["podman", "push"] + tls + [dst], check=True, env=env) def write_image_list(images, path): diff --git a/src/main.py b/src/main.py index fb7d9ca..8c924b1 100644 --- a/src/main.py +++ b/src/main.py @@ -1,8 +1,11 @@ # mirror_tool/main.py import argparse +import os +import sys from helm import pull_chart, render_chart, push_chart, save_chart from images import extract_images, mirror_images, write_image_list from runtime import detect_tools +from auth import registry_host def main(): @@ -34,11 +37,23 @@ def main(): ) parser.add_argument( "--username", - help="Username for registry authentication.", + help="Registry username (or set REGISTRY_USERNAME).", ) parser.add_argument( "--password", - help="Password for registry authentication.", + help="Registry password. Visible in the process list and shell history; " + "prefer --password-stdin or REGISTRY_PASSWORD.", + ) + parser.add_argument( + "--password-stdin", + action="store_true", + help="Read the registry password from the first line of stdin.", + ) + parser.add_argument( + "--auth-registry", + metavar="HOST", + help="Registry the credentials are valid for (default: host of --target-registry). " + "Credentials are never sent to any other registry.", ) parser.add_argument( "--repo-url", @@ -85,9 +100,30 @@ def main(): if not args.save_images and not args.target_registry: parser.error("--target-registry is required unless --save-images is used") + username = args.username or os.environ.get("REGISTRY_USERNAME") + password = os.environ.get("REGISTRY_PASSWORD") + if args.password_stdin: + password = sys.stdin.readline().rstrip("\r\n") + elif args.password is not None: + print("[WARN] --password is visible to other local users; prefer --password-stdin or REGISTRY_PASSWORD.") + password = args.password + if (username is None) != (password is None): + parser.error("registry authentication needs both a username and a password") + if args.ca_cert and not os.path.isfile(args.ca_cert): + parser.error(f"--ca-cert file not found: {args.ca_cert}") + + auth = {"ca_cert": args.ca_cert} + if password is not None: + auth_registry = args.auth_registry or ( + registry_host(args.target_registry) if args.target_registry else None + ) + if not auth_registry: + parser.error("--auth-registry is required when using credentials with --save-images") + auth.update(registry=auth_registry, username=username, password=password) + tools = detect_tools() - chart_path = pull_chart(args.chart, args.version, args.repo_url, args.insecure) + chart_path = pull_chart(args.chart, args.version, args.repo_url, args.insecure, args.ca_cert) rendered = render_chart(chart_path, args.values) images = extract_images(rendered, tools) @@ -101,9 +137,7 @@ def main(): args.dry_run, save_dir=args.images_dir if args.save_images else None, insecure=args.insecure, - ca_cert=args.ca_cert, - username=args.username, - password=args.password, + auth=auth, parallel=args.parallel, skip_existing=args.skip_existing, ) @@ -114,7 +148,7 @@ def main(): if args.save_chart: save_chart(chart_path, args.chart_dir) elif args.push_chart: - push_chart(chart_path, args.target_registry, args.chart_target, args.insecure) + push_chart(chart_path, args.target_registry, args.chart_target, args.insecure, auth) if __name__ == "__main__": From e28caf0ab9957716f646f6846af6f97036750920 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 14:07:44 +0000 Subject: [PATCH 2/4] chore: harden CI/CD, bump dependencies, add tests and retry/exclude options - Split requirements into runtime (pyyaml), build (pyinstaller, setuptools) and dev (pytest, bandit, pip-audit); bump pyyaml 6.0.3, setuptools 84.2.1, pyinstaller 6.22.3. The shipped binary no longer lists bandit & co. - Validate image refs from charts (reject leading '-', whitespace, shell chars) before they become command arguments; parse numbers as strings so 'tag: 1.10' is no longer mirrored as 1.1. - Add --retries (exponential backoff) and --exclude-image. - Add unit tests (22) and a CI workflow: pytest on 3.10/3.13, bandit, pip-audit, PyInstaller smoke build. - Pin all GitHub Actions to commit SHAs at their latest releases, add Dependabot for pip/actions/docker, timeouts, concurrency, persist-credentials. - Docker publish and the 6-hourly rebuild now build -> smoke test -> Trivy gate (fixable HIGH/CRITICAL) -> push, instead of pushing first. The rebuild uses --no-cache so apk upgrade sees new packages. - Binary release: read-only build job, separate release job with contents:write, sha256 checksum attached. - Docs: new flags, development section, SECURITY notes. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013XXYXZzMgKN2pShvj9LqVJ --- .dockerignore | 9 ++- .github/dependabot.yml | 20 ++++++ .github/workflows/binary-build.yml | 62 ++++++++++++---- .github/workflows/ci.yml | 72 +++++++++++++++++++ .github/workflows/docker-publish.yml | 50 +++++++++---- .github/workflows/schedule-rebuild.yml | 48 ++++++++++--- .github/workflows/security-scan.yml | 27 ++++--- Dockerfile | 6 +- README.md | 25 ++++++- SECURITY.md | 7 ++ pytest.ini | 3 + requirements-build.txt | 3 + requirements-dev.txt | 5 ++ requirements.txt | 10 +-- src/images.py | 99 +++++++++++++++++++++----- src/main.py | 21 +++++- tests/test_auth.py | 44 ++++++++++++ tests/test_images.py | 87 ++++++++++++++++++++++ 18 files changed, 516 insertions(+), 82 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 pytest.ini create mode 100644 requirements-build.txt create mode 100644 requirements-dev.txt create mode 100644 tests/test_auth.py create mode 100644 tests/test_images.py diff --git a/.dockerignore b/.dockerignore index ab4f1b6..c555251 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,14 @@ +*.md +*.sarif +*.spec .git .github -*.md __pycache__ build/ dist/ +pytest.ini +requirements-dev.txt test/ +tests/ tmp/ virtenv -*.spec -*.sarif diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7b24dfc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + groups: + python-deps: + patterns: ["*"] + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + - package-ecosystem: docker + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/binary-build.yml b/.github/workflows/binary-build.yml index da7286b..09c64a1 100644 --- a/.github/workflows/binary-build.yml +++ b/.github/workflows/binary-build.yml @@ -5,31 +5,44 @@ on: tags: ["v*"] workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: - contents: write + contents: read security-events: write - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt - - name: Install dependencies - run: pip install -r requirements.txt + - name: Run tests + run: | + pip install -r requirements-dev.txt + python -m pytest -q - name: Build binary with PyInstaller - run: pyinstaller --onefile -n helm-mirror src/main.py + run: | + pip install -r requirements.txt -r requirements-build.txt + pyinstaller --onefile -n helm-mirror src/main.py + ./dist/helm-mirror --help > /dev/null + (cd dist && sha256sum helm-mirror > helm-mirror.sha256) - name: Scan binary with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: fs scan-ref: dist/helm-mirror @@ -39,22 +52,41 @@ jobs: exit-code: "0" - name: Upload Trivy binary scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 if: always() && hashFiles('trivy-binary.sarif') != '' with: sarif_file: trivy-binary.sarif category: trivy-binary - name: Upload binary artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: helm-mirror-linux-amd64 - path: dist/helm-mirror + path: | + dist/helm-mirror + dist/helm-mirror.sha256 if-no-files-found: error + # Only this job can write to the repository, and only for tag builds. + release: + needs: build + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + + steps: + - name: Download binary + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: helm-mirror-linux-amd64 + path: dist + - name: Create GitHub Release - if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: - files: dist/helm-mirror + files: | + dist/helm-mirror + dist/helm-mirror.sha256 generate_release_notes: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..74c7259 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: Tests (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.13"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements-dev.txt + - run: python -m pytest -q + + security: + name: Bandit + dependency audit + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements-dev.txt + # Medium severity and above fails the build; the Low findings are the + # expected "subprocess without shell" notes for a CLI wrapper. + - run: bandit -ll -r src + - run: pip-audit -r requirements.txt -r requirements-build.txt -r requirements-dev.txt + + binary-smoke: + name: Build binary (smoke test) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements.txt -r requirements-build.txt + - run: pyinstaller --onefile -n helm-mirror src/main.py + - run: ./dist/helm-mirror --help diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e6b89d0..a9ff437 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,25 +7,31 @@ on: pull_request: branches: [main] +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write security-events: write - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GitHub Container Registry if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -33,7 +39,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ghcr.io/${{ github.repository }} tags: | @@ -43,30 +49,50 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable={{is_default_branch}} - - name: Build and push - id: build - uses: docker/build-push-action@v6 + # Build locally first; nothing is pushed until the image passed the checks below. + - name: Build image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: ${{ github.event_name != 'pull_request' }} + push: false load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Smoke test image + env: + IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + run: | + docker run --rm "$IMAGE" --help > /dev/null + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'test "$(id -u)" != 0 && helm version --short && crane version && skopeo --version && yq --version' + - name: Scan Docker image with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} format: sarif output: trivy-image.sarif severity: CRITICAL,HIGH - exit-code: "0" + ignore-unfixed: true + # Fail only on vulnerabilities that already have a fix available. + exit-code: "1" - name: Upload Trivy image scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 if: always() && hashFiles('trivy-image.sarif') != '' with: sarif_file: trivy-image.sarif category: trivy-docker-image + + - name: Push image + if: github.event_name != 'pull_request' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha diff --git a/.github/workflows/schedule-rebuild.yml b/.github/workflows/schedule-rebuild.yml index 849b240..7160491 100644 --- a/.github/workflows/schedule-rebuild.yml +++ b/.github/workflows/schedule-rebuild.yml @@ -2,36 +2,64 @@ name: Rebuild Latest Image Schedule on: schedule: - - cron: '0 */6 * * *' # Every six hours + - cron: "0 */6 * * *" # Every six hours workflow_dispatch: +concurrency: + group: schedule-rebuild + cancel-in-progress: false + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image (latest) - uses: docker/build-push-action@v6 + # Base image packages change independently of this repo, so do not reuse + # cached layers; otherwise `apk upgrade` would never see new packages. + - name: Build image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: true + push: false + load: true + pull: true + no-cache: true tags: ghcr.io/${{ github.repository }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + + - name: Smoke test image + run: | + IMAGE=ghcr.io/${{ github.repository }}:latest + docker run --rm "$IMAGE" --help > /dev/null + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'test "$(id -u)" != 0 && helm version --short && crane version && skopeo --version && yq --version' + + # Never replace a working :latest with an image that has a fixable HIGH/CRITICAL CVE. + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ghcr.io/${{ github.repository }}:latest + severity: CRITICAL,HIGH + ignore-unfixed: true + exit-code: "1" + + - name: Push image + run: docker push ghcr.io/${{ github.repository }}:latest diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 39d23c7..cf3497f 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -2,27 +2,33 @@ name: Security Scan on: schedule: - - cron: "0 6 * * 1" # Every Monday at 06:00 UTC + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC push: branches: [main] pull_request: branches: [main] workflow_dispatch: +permissions: + contents: read + jobs: scan-source: name: Scan source code runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read security-events: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Scan source with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: fs scan-ref: . @@ -32,8 +38,8 @@ jobs: exit-code: "0" - name: Upload source scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - if: always() + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + if: always() && hashFiles('trivy-source.sarif') != '' with: sarif_file: trivy-source.sarif category: trivy-source @@ -41,6 +47,7 @@ jobs: scan-image: name: Scan published Docker image runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read security-events: write @@ -50,10 +57,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Scan published image with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ghcr.io/${{ github.repository }}:latest format: sarif @@ -62,8 +71,8 @@ jobs: exit-code: "0" - name: Upload image scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - if: always() + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + if: always() && hashFiles('trivy-image.sarif') != '' with: sarif_file: trivy-image.sarif category: trivy-image-scheduled diff --git a/Dockerfile b/Dockerfile index c4271b6..9780546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apk update && apk add --no-cache \ WORKDIR /app -COPY requirements.txt . -RUN pip3 install --no-cache-dir -r requirements.txt +COPY requirements.txt requirements-build.txt ./ +RUN pip3 install --no-cache-dir -r requirements.txt -r requirements-build.txt COPY src/ src/ @@ -43,4 +43,4 @@ COPY --from=builder /app/dist/helm-mirror /usr/local/bin/helm-mirror USER nonroot ENTRYPOINT ["/usr/local/bin/helm-mirror"] -CMD ["--help"] \ No newline at end of file +CMD ["--help"] diff --git a/README.md b/README.md index 4bf7297..5dbd6b1 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,18 @@ helm-mirror \ --dry-run ``` +**Faster, more robust runs:** + +```bash +helm-mirror \ + --chart bitnami/nginx \ + --target-registry registry.internal.corp \ + --parallel 4 --skip-existing --retries 3 \ + --exclude-image 'docker.io/bitnami/os-shell*' +``` + +`--retries N` retries failed copies with exponential backoff, `--exclude-image` takes a glob and can be repeated, and image references from the chart that look malformed (for example starting with `-`) are ignored instead of being passed to the copy tool. + **Authenticate against the target registry:** ```bash @@ -204,7 +216,7 @@ Security findings are uploaded as SARIF reports to the [GitHub Security tab](htt **Binary:** ```bash -pip install pyinstaller +pip install -r requirements.txt -r requirements-build.txt pyinstaller --onefile -n helm-mirror src/main.py # output: dist/helm-mirror ``` @@ -222,6 +234,17 @@ See [SECURITY.md](SECURITY.md) for the vulnerability reporting policy and detail The latest Docker image is automatically rebuilt every six hours to incorporate the latest security patches from the base image and dependencies. +## Development + +```bash +pip install -r requirements-dev.txt +python -m pytest # unit tests +bandit -ll -r src # static security checks +pip-audit -r requirements.txt -r requirements-build.txt -r requirements-dev.txt +``` + +CI runs these on every pull request. GitHub Actions and the Docker base image are updated by Dependabot; all actions are pinned to commit SHAs. + ## License Apache License 2.0. See LICENSE for details. diff --git a/SECURITY.md b/SECURITY.md index 0050a76..747e53c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -49,3 +49,10 @@ as SARIF reports, where they can be reviewed and tracked. GitHub Actions runners and published directly as a GitHub Release asset. - All CI/CD workflows use pinned action versions and rely only on `GITHUB_TOKEN` with least-privilege permissions — no external secrets are required. + +- Registry credentials are written to a private temporary config that only contains the target registry + (never the source registries) and are removed after each command. Prefer `--password-stdin` or + `REGISTRY_PASSWORD` over `--password`. +- Image references taken from charts are validated before they are used as command arguments. +- The Docker image and the scheduled `latest` rebuild are only pushed after a smoke test and a Trivy scan + without fixable HIGH/CRITICAL findings. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..442bb94 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = src +testpaths = tests diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 0000000..b96209d --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,3 @@ +# Build tooling for the standalone binary (not shipped) +setuptools==84.2.1 +pyinstaller==6.22.3 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..8790602 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Tests and security linting +-r requirements.txt +pytest==9.2.0 +bandit==1.9.4 +pip-audit==2.10.1 diff --git a/requirements.txt b/requirements.txt index 28170ce..b03e7d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,2 @@ -# Runtime dependencies -pyyaml==6.0.2 - -# Build/Linting tools (Pinned for consistency) -setuptools==78.1.1 -#Nuitka==4.0.5 -bandit==1.9.4 -pyinstaller==6.20.0 +# Runtime dependencies (the only packages bundled into the binary) +pyyaml==6.0.3 diff --git a/src/images.py b/src/images.py index a30a896..6542feb 100644 --- a/src/images.py +++ b/src/images.py @@ -2,12 +2,33 @@ import subprocess import re import os +import time +import fnmatch import yaml from auth import registry_env +class _StringLoader(yaml.SafeLoader): + """SafeLoader that keeps numbers as strings. + + Otherwise an unquoted `tag: 1.10` is parsed as the float 1.1 and the wrong + image tag would be mirrored. + """ + + +_StringLoader.yaml_implicit_resolvers = { + ch: [(tag, rx) for tag, rx in resolvers + if tag not in ("tag:yaml.org,2002:float", "tag:yaml.org,2002:int")] + for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} + +# Image refs come from untrusted charts and end up as command arguments, so +# only accept plausible references (this also rejects values starting with "-"). +_IMAGE_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/:@+-]*$") + + def _walk_for_images(node, images): """Recursively walk a parsed YAML node and collect image references. @@ -51,11 +72,37 @@ def _walk_for_images(node, images): def extract_images(rendered_yaml, tools): + return _filter_valid(_extract_images(rendered_yaml, tools)) + + +def _filter_valid(images): + valid = set() + for img in images: + if _IMAGE_REF.match(img): + valid.add(img) + else: + print(f"[WARN] Ignoring invalid image reference: {img!r}") + return valid + + +def filter_excluded(images, patterns): + """Drop images matching any fnmatch pattern (e.g. 'docker.io/bitnami/*').""" + kept = set() + for img in images: + hit = next((p for p in patterns if fnmatch.fnmatch(img, p)), None) + if hit: + print(f"[SKIP] {img} (excluded by {hit!r})") + else: + kept.add(img) + return kept + + +def _extract_images(rendered_yaml, tools): images = set() # Try YAML parsing first — handles all known Helm image reference patterns try: - docs = list(yaml.safe_load_all(rendered_yaml)) + docs = list(yaml.load_all(rendered_yaml, Loader=_StringLoader)) _walk_for_images(docs, images) print("[INFO] Using YAML parsing for image extraction") return images @@ -90,6 +137,19 @@ def extract_images(rendered_yaml, tools): return images +def _run_retry(cmd, env, retries): + """Run a command, retrying transient failures with exponential backoff.""" + for attempt in range(retries + 1): + try: + return subprocess.run(cmd, check=True, env=env) + except subprocess.CalledProcessError: + if attempt == retries: + raise + delay = 2 ** (attempt + 1) + print(f"[WARN] {cmd[0]} {cmd[1]} failed, retrying in {delay}s ({attempt + 1}/{retries})") + time.sleep(delay) + + def _image_exists(image, tools, insecure=False, auth=None): """Return True when the image already exists in the registry.""" tool = tools.get("copy") @@ -118,13 +178,14 @@ def mirror_images( auth=None, parallel=1, skip_existing=False, + retries=0, ): succeeded = [] failed = [] def _process(img): if save_dir is not None: - tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, auth) + tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, auth, retries) print(f"[INFO] {img} -> {tar_path}") return tar_path @@ -134,7 +195,7 @@ def _process(img): return new print(f"[INFO] {img} -> {new}") if not dry_run: - copy_image(img, new, tools, insecure, auth) + copy_image(img, new, tools, insecure, auth, retries) return new if parallel > 1: @@ -198,7 +259,7 @@ def _warn_docker_insecure(): ) -def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, auth=None): +def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, auth=None, retries=0): os.makedirs(output_dir, exist_ok=True) tar_path = os.path.join(output_dir, _image_to_filename(image)) @@ -210,44 +271,44 @@ def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, a with registry_env(tool, **(auth or {})) as env: if tool == "crane": cmd = ["crane", "pull"] + (["--insecure"] if insecure else []) - subprocess.run(cmd + ["--format=tarball", image, tar_path], check=True, env=env) + _run_retry(cmd + ["--format=tarball", image, tar_path], env, retries) elif tool == "skopeo": cmd = ["skopeo", "copy"] + (["--src-tls-verify=false"] if insecure else []) - subprocess.run(cmd + [f"docker://{image}", f"docker-archive:{tar_path}"], check=True, env=env) + _run_retry(cmd + [f"docker://{image}", f"docker-archive:{tar_path}"], env, retries) elif tool == "docker": if insecure: _warn_docker_insecure() - subprocess.run(["docker", "pull", image], check=True, env=env) - subprocess.run(["docker", "save", image, "-o", tar_path], check=True, env=env) + _run_retry(["docker", "pull", image], env, retries) + _run_retry(["docker", "save", image, "-o", tar_path], env, retries) else: - subprocess.run(["podman", "pull"] + tls + [image], check=True, env=env) - subprocess.run(["podman", "save", image, "-o", tar_path], check=True, env=env) + _run_retry(["podman", "pull"] + tls + [image], env, retries) + _run_retry(["podman", "save", image, "-o", tar_path], env, retries) return tar_path -def copy_image(src, dst, tools, insecure=False, auth=None): +def copy_image(src, dst, tools, insecure=False, auth=None, retries=0): tool = tools["copy"] tls = ["--tls-verify=false"] if insecure else [] with registry_env(tool, **(auth or {})) as env: if tool == "crane": cmd = ["crane", "copy"] + (["--insecure"] if insecure else []) - subprocess.run(cmd + [src, dst], check=True, env=env) + _run_retry(cmd + [src, dst], env, retries) elif tool == "skopeo": cmd = ["skopeo", "copy"] if insecure: cmd += ["--src-tls-verify=false", "--dest-tls-verify=false"] - subprocess.run(cmd + [f"docker://{src}", f"docker://{dst}"], check=True, env=env) + _run_retry(cmd + [f"docker://{src}", f"docker://{dst}"], env, retries) elif tool == "docker": if insecure: _warn_docker_insecure() - subprocess.run(["docker", "pull", src], check=True, env=env) - subprocess.run(["docker", "tag", src, dst], check=True, env=env) - subprocess.run(["docker", "push", dst], check=True, env=env) + _run_retry(["docker", "pull", src], env, retries) + _run_retry(["docker", "tag", src, dst], env, retries) + _run_retry(["docker", "push", dst], env, retries) else: - subprocess.run(["podman", "pull"] + tls + [src], check=True, env=env) - subprocess.run(["podman", "tag", src, dst], check=True, env=env) - subprocess.run(["podman", "push"] + tls + [dst], check=True, env=env) + _run_retry(["podman", "pull"] + tls + [src], env, retries) + _run_retry(["podman", "tag", src, dst], env, retries) + _run_retry(["podman", "push"] + tls + [dst], env, retries) def write_image_list(images, path): diff --git a/src/main.py b/src/main.py index 8c924b1..7c574c2 100644 --- a/src/main.py +++ b/src/main.py @@ -3,7 +3,7 @@ import os import sys from helm import pull_chart, render_chart, push_chart, save_chart -from images import extract_images, mirror_images, write_image_list +from images import extract_images, filter_excluded, mirror_images, write_image_list from runtime import detect_tools from auth import registry_host @@ -89,6 +89,20 @@ def main(): metavar="N", help="Number of images to mirror concurrently (default: 1).", ) + parser.add_argument( + "--retries", + type=int, + default=2, + metavar="N", + help="Retry failed image copies N times with exponential backoff (default: 2).", + ) + parser.add_argument( + "--exclude-image", + action="append", + default=[], + metavar="PATTERN", + help="Skip images matching this glob (repeatable), e.g. 'docker.io/bitnami/*'.", + ) parser.add_argument( "--skip-existing", action="store_true", @@ -109,6 +123,8 @@ def main(): password = args.password if (username is None) != (password is None): parser.error("registry authentication needs both a username and a password") + if args.parallel < 1 or args.retries < 0: + parser.error("--parallel must be >= 1 and --retries >= 0") if args.ca_cert and not os.path.isfile(args.ca_cert): parser.error(f"--ca-cert file not found: {args.ca_cert}") @@ -126,7 +142,7 @@ def main(): chart_path = pull_chart(args.chart, args.version, args.repo_url, args.insecure, args.ca_cert) rendered = render_chart(chart_path, args.values) - images = extract_images(rendered, tools) + images = filter_excluded(extract_images(rendered, tools), args.exclude_image) print(f"[INFO] Found {len(images)} images") mirrored = mirror_images( @@ -140,6 +156,7 @@ def main(): auth=auth, parallel=args.parallel, skip_existing=args.skip_existing, + retries=args.retries, ) if args.image_list_file: diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..2f1515f --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,44 @@ +import json +import os + +import pytest + +from auth import registry_env, registry_host + + +def test_registry_host(): + assert registry_host("reg.example.com:5000/prefix") == "reg.example.com:5000" + assert registry_host("https://reg.example.com") == "reg.example.com" + + +def test_credentials_only_for_target_and_private(): + with registry_env("crane", "reg.example.com", "u", "p") as env: + cfg = os.path.join(env["DOCKER_CONFIG"], "config.json") + assert oct(os.stat(cfg).st_mode & 0o777) == "0o600" + assert list(json.load(open(cfg))["auths"]) == ["reg.example.com"] + tmp = env["DOCKER_CONFIG"] + assert not os.path.exists(tmp) + + +def test_docker_hub_key(): + with registry_env("crane", "docker.io", "u", "p") as env: + auths = json.load(open(env["REGISTRY_AUTH_FILE"]))["auths"] + assert list(auths) == ["https://index.docker.io/v1/"] + + +def test_password_requires_username_and_registry(): + with pytest.raises(ValueError): + with registry_env("crane", "reg", None, "p"): + pass + + +def test_ca_cert_is_appended_not_replacing(tmp_path): + ca = tmp_path / "ca.pem" + ca.write_text("-----BEGIN CERTIFICATE-----\nCUSTOM\n-----END CERTIFICATE-----\n") + with registry_env("skopeo", ca_cert=str(ca)) as env: + assert "CUSTOM" in open(env["SSL_CERT_FILE"]).read() + + +def test_no_credentials_leaves_env_untouched(): + with registry_env("crane") as env: + assert env.get("DOCKER_CONFIG") == os.environ.get("DOCKER_CONFIG") diff --git a/tests/test_images.py b/tests/test_images.py new file mode 100644 index 0000000..32e83ed --- /dev/null +++ b/tests/test_images.py @@ -0,0 +1,87 @@ +import subprocess + +import pytest + +import images + + +def extract(doc): + return images.extract_images(doc, {}) + + +def test_standard_image_string(): + assert extract("image: nginx:1.27") == {"nginx:1.27"} + + +def test_nested_repository_and_numeric_tag_is_not_mangled(): + doc = "image:\n repository: quay.io/foo/bar\n tag: 1.10\n" + assert extract(doc) == {"quay.io/foo/bar:1.10"} + + +def test_sibling_repository_image_version(): + doc = "repository: nvcr.io/nvidia\nimage: gpu-operator\nversion: v1.2\n" + assert extract(doc) == {"nvcr.io/nvidia/gpu-operator:v1.2"} + + +@pytest.mark.parametrize("bad", ["--insecure", "-x", "a b", "img;rm", "$(id)"]) +def test_invalid_refs_are_dropped(bad): + assert extract(f"image: '{bad}'") == set() + + +def test_exclude_patterns(): + imgs = {"docker.io/bitnami/nginx:1", "quay.io/a/b:2"} + assert images.filter_excluded(imgs, ["docker.io/bitnami/*"]) == {"quay.io/a/b:2"} + + +@pytest.mark.parametrize( + "image,expected", + [ + ("docker.io/library/nginx:1", "reg.io/mirror/library/nginx:1"), + ("nginx:1", "reg.io/mirror/nginx:1"), + ("quay.io/a/b@sha256:abc", "reg.io/mirror/a/b@sha256:abc"), + ], +) +def test_rewrite_image(image, expected): + assert images.rewrite_image(image, "reg.io", "mirror") == expected + + +def test_tar_filename_has_no_path_separators(): + assert "/" not in images._image_to_filename("a/b/../c:1") + + +def test_run_retry_retries_then_succeeds(monkeypatch): + calls = [] + + def fake_run(cmd, check, env): + calls.append(cmd) + if len(calls) < 3: + raise subprocess.CalledProcessError(1, cmd) + + monkeypatch.setattr(images.subprocess, "run", fake_run) + monkeypatch.setattr(images.time, "sleep", lambda s: None) + images._run_retry(["crane", "copy", "a", "b"], {}, retries=2) + assert len(calls) == 3 + + +def test_run_retry_gives_up(monkeypatch): + def fake_run(cmd, check, env): + raise subprocess.CalledProcessError(1, cmd) + + monkeypatch.setattr(images.subprocess, "run", fake_run) + monkeypatch.setattr(images.time, "sleep", lambda s: None) + with pytest.raises(subprocess.CalledProcessError): + images._run_retry(["crane", "copy", "a", "b"], {}, retries=1) + + +def test_mirror_images_reports_failures_but_attempts_all(monkeypatch): + seen = [] + + def fake_copy(src, dst, tools, insecure, auth, retries): + seen.append(src) + if src == "bad:1": + raise RuntimeError("boom") + + monkeypatch.setattr(images, "copy_image", fake_copy) + with pytest.raises(RuntimeError, match="1 image"): + images.mirror_images(["ok:1", "bad:1", "ok:2"], "r.io", "", {"copy": "crane"}) + assert sorted(seen) == ["bad:1", "ok:1", "ok:2"] From f4bf2dc2ef910b6fdff813429e634a1e00c7ae75 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:03:51 +0000 Subject: [PATCH 3/4] ci: refresh apk layer on every CI build and print the Trivy gate findings The Docker PR build reused the cached 'apk upgrade' layer, so the Trivy gate could flag packages that a fresh build would already have fixed. Pass the run id as APK_REFRESH build arg, and split the Trivy step into a SARIF report and a table-format gate so a failure shows which vulnerabilities caused it. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013XXYXZzMgKN2pShvj9LqVJ --- .github/workflows/docker-publish.yml | 18 ++++++++++++++---- Dockerfile | 6 +++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a9ff437..575a5da 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -58,6 +58,7 @@ jobs: load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: APK_REFRESH=${{ github.run_id }} cache-from: type=gha cache-to: type=gha,mode=max @@ -69,16 +70,14 @@ jobs: docker run --rm --entrypoint sh "$IMAGE" -c \ 'test "$(id -u)" != 0 && helm version --short && crane version && skopeo --version && yq --version' - - name: Scan Docker image with Trivy + - name: Scan Docker image with Trivy (SARIF report) uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} format: sarif output: trivy-image.sarif severity: CRITICAL,HIGH - ignore-unfixed: true - # Fail only on vulnerabilities that already have a fix available. - exit-code: "1" + exit-code: "0" - name: Upload Trivy image scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 @@ -87,6 +86,17 @@ jobs: sarif_file: trivy-image.sarif category: trivy-docker-image + # Gate: fail only on vulnerabilities that already have a fix available, + # and print them to the log so the failure is actionable. + - name: Fail on fixable HIGH/CRITICAL vulnerabilities + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + format: table + severity: CRITICAL,HIGH + ignore-unfixed: true + exit-code: "1" + - name: Push image if: github.event_name != 'pull_request' uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 diff --git a/Dockerfile b/Dockerfile index 9780546..8bfbe33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,12 @@ RUN pyinstaller --onefile -n helm-mirror src/main.py # Stage 2: Final image - Contains only the runtime dependencies and the built binary FROM docker.io/chainguard/wolfi-base +# Changing APK_REFRESH (CI passes the run id) invalidates this layer so that +# `apk upgrade` is never served from a stale build cache. +ARG APK_REFRESH=0 + # Remove version pins from world file to allow upgrades, then upgrade all packages -RUN apk update && \ +RUN : "${APK_REFRESH}" && apk update && \ # Remove specific version pins for security-critical packages to allow upgrades sed -i '/^glibc=/d; /^glibc-locale-posix=/d; /^ld-linux=/d; /^libcrypt1=/d; /^libcrypto3=/d; /^libssl3=/d' /etc/apk/world && \ apk upgrade -a && \ From c9e07fe92552a7412c5edabca6f19d4e6f7d6fd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:05:59 +0000 Subject: [PATCH 4/4] fix: drop skopeo from the Docker image crane is always present in the image and is preferred by tool detection, so skopeo was never used there. It was the only package behind the 13 fixable HIGH Go-library findings that made the Trivy gate fail. skopeo remains supported as a copy tool on host installs. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013XXYXZzMgKN2pShvj9LqVJ --- .github/workflows/docker-publish.yml | 2 +- .github/workflows/schedule-rebuild.yml | 2 +- Dockerfile | 3 +-- README.md | 2 +- SECURITY.md | 3 ++- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 575a5da..1e4d009 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -68,7 +68,7 @@ jobs: run: | docker run --rm "$IMAGE" --help > /dev/null docker run --rm --entrypoint sh "$IMAGE" -c \ - 'test "$(id -u)" != 0 && helm version --short && crane version && skopeo --version && yq --version' + 'test "$(id -u)" != 0 && helm version --short && crane version && yq --version' - name: Scan Docker image with Trivy (SARIF report) uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 diff --git a/.github/workflows/schedule-rebuild.yml b/.github/workflows/schedule-rebuild.yml index 7160491..7cfd189 100644 --- a/.github/workflows/schedule-rebuild.yml +++ b/.github/workflows/schedule-rebuild.yml @@ -50,7 +50,7 @@ jobs: IMAGE=ghcr.io/${{ github.repository }}:latest docker run --rm "$IMAGE" --help > /dev/null docker run --rm --entrypoint sh "$IMAGE" -c \ - 'test "$(id -u)" != 0 && helm version --short && crane version && skopeo --version && yq --version' + 'test "$(id -u)" != 0 && helm version --short && crane version && yq --version' # Never replace a working :latest with an image that has a fixable HIGH/CRITICAL CVE. - name: Scan image with Trivy diff --git a/Dockerfile b/Dockerfile index 8bfbe33..c8fe1d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,8 +38,7 @@ RUN : "${APK_REFRESH}" && apk update && \ apk add --no-cache \ helm \ yq \ - crane \ - skopeo && \ + crane && \ rm -rf /var/cache/apk/* COPY --from=builder /app/dist/helm-mirror /usr/local/bin/helm-mirror diff --git a/README.md b/README.md index 5dbd6b1..cfdf5cd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Designed for air-gapped environments or maintaining a local cache of external de | Helm | `^3.14` | Required | | yq | `^4.40` | Recommended — falls back to regex without it | | crane | `^0.19` | Preferred copy tool | -| skopeo | `^1.13` | Second choice — daemonless, no container runtime required | +| skopeo | `^1.13` | Second choice — daemonless, no container runtime required (not included in the Docker image, which uses `crane`) | | docker / podman | any | Fallback copy tools (require running daemon) ## Installation diff --git a/SECURITY.md b/SECURITY.md index 747e53c..cf0e82d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -44,7 +44,8 @@ as SARIF reports, where they can be reviewed and tracked. All installed packages are upgraded to their latest patched versions during the build. `kubectl` is intentionally excluded from the image since it is not used by the tool, reducing the attack surface. - The image includes `skopeo` as the preferred daemonless alternative to `docker`/`podman`. + `skopeo` is intentionally not part of the image: `crane` covers all copy operations there and `skopeo` + only added unused attack surface. On a host install, `skopeo` is still supported as a copy tool. - The standalone Linux binary is built with [PyInstaller](https://pyinstaller.org/) on `ubuntu-latest` GitHub Actions runners and published directly as a GitHub Release asset. - All CI/CD workflows use pinned action versions and rely only on `GITHUB_TOKEN`