diff --git a/.dockerignore b/.dockerignore index ab4f1b6..c555251 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,14 @@ +*.md +*.sarif +*.spec .git .github -*.md __pycache__ build/ dist/ +pytest.ini +requirements-dev.txt test/ +tests/ tmp/ virtenv -*.spec -*.sarif diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7b24dfc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + groups: + python-deps: + patterns: ["*"] + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + - package-ecosystem: docker + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/binary-build.yml b/.github/workflows/binary-build.yml index da7286b..09c64a1 100644 --- a/.github/workflows/binary-build.yml +++ b/.github/workflows/binary-build.yml @@ -5,31 +5,44 @@ on: tags: ["v*"] workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: - contents: write + contents: read security-events: write - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt - - name: Install dependencies - run: pip install -r requirements.txt + - name: Run tests + run: | + pip install -r requirements-dev.txt + python -m pytest -q - name: Build binary with PyInstaller - run: pyinstaller --onefile -n helm-mirror src/main.py + run: | + pip install -r requirements.txt -r requirements-build.txt + pyinstaller --onefile -n helm-mirror src/main.py + ./dist/helm-mirror --help > /dev/null + (cd dist && sha256sum helm-mirror > helm-mirror.sha256) - name: Scan binary with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: fs scan-ref: dist/helm-mirror @@ -39,22 +52,41 @@ jobs: exit-code: "0" - name: Upload Trivy binary scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 if: always() && hashFiles('trivy-binary.sarif') != '' with: sarif_file: trivy-binary.sarif category: trivy-binary - name: Upload binary artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: helm-mirror-linux-amd64 - path: dist/helm-mirror + path: | + dist/helm-mirror + dist/helm-mirror.sha256 if-no-files-found: error + # Only this job can write to the repository, and only for tag builds. + release: + needs: build + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + + steps: + - name: Download binary + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: helm-mirror-linux-amd64 + path: dist + - name: Create GitHub Release - if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: - files: dist/helm-mirror + files: | + dist/helm-mirror + dist/helm-mirror.sha256 generate_release_notes: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..74c7259 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: Tests (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.13"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements-dev.txt + - run: python -m pytest -q + + security: + name: Bandit + dependency audit + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements-dev.txt + # Medium severity and above fails the build; the Low findings are the + # expected "subprocess without shell" notes for a CLI wrapper. + - run: bandit -ll -r src + - run: pip-audit -r requirements.txt -r requirements-build.txt -r requirements-dev.txt + + binary-smoke: + name: Build binary (smoke test) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements*.txt + - run: pip install -r requirements.txt -r requirements-build.txt + - run: pyinstaller --onefile -n helm-mirror src/main.py + - run: ./dist/helm-mirror --help diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e6b89d0..1e4d009 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,25 +7,31 @@ on: pull_request: branches: [main] +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write security-events: write - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GitHub Container Registry if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -33,7 +39,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ghcr.io/${{ github.repository }} tags: | @@ -43,20 +49,29 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable={{is_default_branch}} - - name: Build and push - id: build - uses: docker/build-push-action@v6 + # Build locally first; nothing is pushed until the image passed the checks below. + - name: Build image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: ${{ github.event_name != 'pull_request' }} + push: false load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: APK_REFRESH=${{ github.run_id }} cache-from: type=gha cache-to: type=gha,mode=max - - name: Scan Docker image with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + - name: Smoke test image + env: + IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + run: | + docker run --rm "$IMAGE" --help > /dev/null + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'test "$(id -u)" != 0 && helm version --short && crane version && yq --version' + + - name: Scan Docker image with Trivy (SARIF report) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} format: sarif @@ -65,8 +80,29 @@ jobs: exit-code: "0" - name: Upload Trivy image scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 if: always() && hashFiles('trivy-image.sarif') != '' with: sarif_file: trivy-image.sarif category: trivy-docker-image + + # Gate: fail only on vulnerabilities that already have a fix available, + # and print them to the log so the failure is actionable. + - name: Fail on fixable HIGH/CRITICAL vulnerabilities + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + format: table + severity: CRITICAL,HIGH + ignore-unfixed: true + exit-code: "1" + + - name: Push image + if: github.event_name != 'pull_request' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha diff --git a/.github/workflows/schedule-rebuild.yml b/.github/workflows/schedule-rebuild.yml index 3835c46..7cfd189 100644 --- a/.github/workflows/schedule-rebuild.yml +++ b/.github/workflows/schedule-rebuild.yml @@ -2,37 +2,64 @@ name: Rebuild Latest Image Schedule on: schedule: - - cron: '0 */6 * * *' # Every six hours + - cron: "0 */6 * * *" # Every six hours workflow_dispatch: +concurrency: + group: schedule-rebuild + cancel-in-progress: false + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write - # needed for GHCR login - id-token: none steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image (latest) - uses: docker/build-push-action@v6 + # Base image packages change independently of this repo, so do not reuse + # cached layers; otherwise `apk upgrade` would never see new packages. + - name: Build image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - push: true + push: false + load: true + pull: true + no-cache: true tags: ghcr.io/${{ github.repository }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + + - name: Smoke test image + run: | + IMAGE=ghcr.io/${{ github.repository }}:latest + docker run --rm "$IMAGE" --help > /dev/null + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'test "$(id -u)" != 0 && helm version --short && crane version && yq --version' + + # Never replace a working :latest with an image that has a fixable HIGH/CRITICAL CVE. + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ghcr.io/${{ github.repository }}:latest + severity: CRITICAL,HIGH + ignore-unfixed: true + exit-code: "1" + + - name: Push image + run: docker push ghcr.io/${{ github.repository }}:latest diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 39d23c7..cf3497f 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -2,27 +2,33 @@ name: Security Scan on: schedule: - - cron: "0 6 * * 1" # Every Monday at 06:00 UTC + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC push: branches: [main] pull_request: branches: [main] workflow_dispatch: +permissions: + contents: read + jobs: scan-source: name: Scan source code runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read security-events: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Scan source with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: fs scan-ref: . @@ -32,8 +38,8 @@ jobs: exit-code: "0" - name: Upload source scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - if: always() + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + if: always() && hashFiles('trivy-source.sarif') != '' with: sarif_file: trivy-source.sarif category: trivy-source @@ -41,6 +47,7 @@ jobs: scan-image: name: Scan published Docker image runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read security-events: write @@ -50,10 +57,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Scan published image with Trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: ghcr.io/${{ github.repository }}:latest format: sarif @@ -62,8 +71,8 @@ jobs: exit-code: "0" - name: Upload image scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - if: always() + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + if: always() && hashFiles('trivy-image.sarif') != '' with: sarif_file: trivy-image.sarif category: trivy-image-scheduled diff --git a/Dockerfile b/Dockerfile index c4271b6..c8fe1d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apk update && apk add --no-cache \ WORKDIR /app -COPY requirements.txt . -RUN pip3 install --no-cache-dir -r requirements.txt +COPY requirements.txt requirements-build.txt ./ +RUN pip3 install --no-cache-dir -r requirements.txt -r requirements-build.txt COPY src/ src/ @@ -26,16 +26,19 @@ RUN pyinstaller --onefile -n helm-mirror src/main.py # Stage 2: Final image - Contains only the runtime dependencies and the built binary FROM docker.io/chainguard/wolfi-base +# Changing APK_REFRESH (CI passes the run id) invalidates this layer so that +# `apk upgrade` is never served from a stale build cache. +ARG APK_REFRESH=0 + # Remove version pins from world file to allow upgrades, then upgrade all packages -RUN apk update && \ +RUN : "${APK_REFRESH}" && apk update && \ # Remove specific version pins for security-critical packages to allow upgrades sed -i '/^glibc=/d; /^glibc-locale-posix=/d; /^ld-linux=/d; /^libcrypt1=/d; /^libcrypto3=/d; /^libssl3=/d' /etc/apk/world && \ apk upgrade -a && \ apk add --no-cache \ helm \ yq \ - crane \ - skopeo && \ + crane && \ rm -rf /var/cache/apk/* COPY --from=builder /app/dist/helm-mirror /usr/local/bin/helm-mirror @@ -43,4 +46,4 @@ COPY --from=builder /app/dist/helm-mirror /usr/local/bin/helm-mirror USER nonroot ENTRYPOINT ["/usr/local/bin/helm-mirror"] -CMD ["--help"] \ No newline at end of file +CMD ["--help"] diff --git a/README.md b/README.md index 72baf08..cfdf5cd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Designed for air-gapped environments or maintaining a local cache of external de | Helm | `^3.14` | Required | | yq | `^4.40` | Recommended — falls back to regex without it | | crane | `^0.19` | Preferred copy tool | -| skopeo | `^1.13` | Second choice — daemonless, no container runtime required | +| skopeo | `^1.13` | Second choice — daemonless, no container runtime required (not included in the Docker image, which uses `crane`) | | docker / podman | any | Fallback copy tools (require running daemon) ## Installation @@ -111,11 +111,34 @@ helm-mirror \ --dry-run ``` +**Faster, more robust runs:** + +```bash +helm-mirror \ + --chart bitnami/nginx \ + --target-registry registry.internal.corp \ + --parallel 4 --skip-existing --retries 3 \ + --exclude-image 'docker.io/bitnami/os-shell*' +``` + +`--retries N` retries failed copies with exponential backoff, `--exclude-image` takes a glob and can be repeated, and image references from the chart that look malformed (for example starting with `-`) are ignored instead of being passed to the copy tool. + +**Authenticate against the target registry:** + +```bash +echo "$REGISTRY_PASSWORD" | helm-mirror \ + --chart bitnami/nginx \ + --target-registry registry.internal.corp \ + --username ci-bot --password-stdin \ + --ca-cert /etc/ssl/internal-ca.pem +``` + +Credentials can also come from `REGISTRY_USERNAME` / `REGISTRY_PASSWORD`. Avoid `--password`, which is visible in the process list. Credentials are only valid for the host of `--target-registry` (override with `--auth-registry`) and are never sent to the source registries. `--ca-cert` is added on top of the system trust store (not supported for `docker`/`podman`). + **Run via Docker:** ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ ghcr.io/thenomadbeyond/helm-mirror-cli:latest \ --chart bitnami/nginx \ --target-registry registry.internal.corp \ @@ -129,13 +152,12 @@ When running the Docker image, you may need to mount volumes for: - Output directories (for `--save-images`/`--images-dir` and `--save-chart`/`--chart-dir`) - Temporary storage (if the default `/tmp` is not writable or you want to persist temporary files) -The image runs as a nonroot user (UID 1000). Ensure that any mounted directories are writable by this user. +The image runs as a nonroot user (UID 65532). Ensure that any mounted directories are writable by this user. ### Example: Using Local Charts and Saving Output ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ -v $(pwd)/local-charts:/charts:ro \ -v $(pwd)/output:/output:rw \ -v $(pwd)/tmp:/tmp:rw \ @@ -161,7 +183,6 @@ If you prefer not to mount `/tmp`, you can set the `TMPDIR` environment variable ```bash docker run --rm \ - -v ~/.config/helm:/root/.config/helm:ro \ -v $(pwd)/local-charts:/charts:ro \ -v $(pwd)/output:/output:rw \ -v $(pwd)/tmp:/tmp:rw \ @@ -177,7 +198,7 @@ docker run --rm \ Note: The image already uses `/tmp` as the default temporary directory, so mounting a host directory to `/tmp` is often sufficient. ### Important Notes -- The nonroot user in the container has UID 1000. If you get permission errors, ensure the host directories are writable by UID 1000 (e.g., `chmod a+rw` or change ownership to 1000:1000). +- The nonroot user in the container has UID 65532. If you get permission errors, ensure the host directories are writable by UID 65532 (e.g., `chmod a+rw` or change ownership to 65532:65532). - For reading local charts, the mount can be read-only (`:ro`). - For writing output or temporary files, the mount must be read-write (`:rw`). ## CI/CD Pipelines @@ -195,7 +216,7 @@ Security findings are uploaded as SARIF reports to the [GitHub Security tab](htt **Binary:** ```bash -pip install pyinstaller +pip install -r requirements.txt -r requirements-build.txt pyinstaller --onefile -n helm-mirror src/main.py # output: dist/helm-mirror ``` @@ -213,6 +234,17 @@ See [SECURITY.md](SECURITY.md) for the vulnerability reporting policy and detail The latest Docker image is automatically rebuilt every six hours to incorporate the latest security patches from the base image and dependencies. +## Development + +```bash +pip install -r requirements-dev.txt +python -m pytest # unit tests +bandit -ll -r src # static security checks +pip-audit -r requirements.txt -r requirements-build.txt -r requirements-dev.txt +``` + +CI runs these on every pull request. GitHub Actions and the Docker base image are updated by Dependabot; all actions are pinned to commit SHAs. + ## License Apache License 2.0. See LICENSE for details. diff --git a/SECURITY.md b/SECURITY.md index 0050a76..cf0e82d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -44,8 +44,16 @@ as SARIF reports, where they can be reviewed and tracked. All installed packages are upgraded to their latest patched versions during the build. `kubectl` is intentionally excluded from the image since it is not used by the tool, reducing the attack surface. - The image includes `skopeo` as the preferred daemonless alternative to `docker`/`podman`. + `skopeo` is intentionally not part of the image: `crane` covers all copy operations there and `skopeo` + only added unused attack surface. On a host install, `skopeo` is still supported as a copy tool. - The standalone Linux binary is built with [PyInstaller](https://pyinstaller.org/) on `ubuntu-latest` GitHub Actions runners and published directly as a GitHub Release asset. - All CI/CD workflows use pinned action versions and rely only on `GITHUB_TOKEN` with least-privilege permissions — no external secrets are required. + +- Registry credentials are written to a private temporary config that only contains the target registry + (never the source registries) and are removed after each command. Prefer `--password-stdin` or + `REGISTRY_PASSWORD` over `--password`. +- Image references taken from charts are validated before they are used as command arguments. +- The Docker image and the scheduled `latest` rebuild are only pushed after a smoke test and a Trivy scan + without fixable HIGH/CRITICAL findings. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..442bb94 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = src +testpaths = tests diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 0000000..b96209d --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,3 @@ +# Build tooling for the standalone binary (not shipped) +setuptools==84.2.1 +pyinstaller==6.22.3 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..8790602 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Tests and security linting +-r requirements.txt +pytest==9.2.0 +bandit==1.9.4 +pip-audit==2.10.1 diff --git a/requirements.txt b/requirements.txt index 28170ce..b03e7d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,2 @@ -# Runtime dependencies -pyyaml==6.0.2 - -# Build/Linting tools (Pinned for consistency) -setuptools==78.1.1 -#Nuitka==4.0.5 -bandit==1.9.4 -pyinstaller==6.20.0 +# Runtime dependencies (the only packages bundled into the binary) +pyyaml==6.0.3 diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..e9f0bc5 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,82 @@ +# mirror_tool/auth.py +import contextlib +import json +import os +import shutil +import ssl +import tempfile +from base64 import b64encode + +_DOCKER_HUB = "https://index.docker.io/v1/" +_HUB_HOSTS = {"docker.io", "index.docker.io", "registry-1.docker.io"} +_SYSTEM_CA_FALLBACKS = ( + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", +) + + +def registry_host(target): + """Host[:port] of a target registry string such as host:5000/prefix.""" + return target.split("://", 1)[-1].split("/", 1)[0] + + +def _system_ca_bundle(): + for path in (ssl.get_default_verify_paths().cafile, *_SYSTEM_CA_FALLBACKS): + if path and os.path.isfile(path): + return path + return None + + +def _write_private(path, text): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as f: + f.write(text) + + +@contextlib.contextmanager +def registry_env(tool, registry=None, username=None, password=None, ca_cert=None): + """Yield a subprocess environment carrying registry credentials / a CA. + + Credentials are written to a private temp config that only contains an + entry for `registry`, so they are never offered to any other host (e.g. + the upstream source registry). The temp dir is always removed on exit. + """ + env = os.environ.copy() + tmp = tempfile.mkdtemp(prefix="helm-mirror-") + try: + if password is not None: + if not (username and registry): + raise ValueError("credentials need both a username and a registry") + key = _DOCKER_HUB if registry in _HUB_HOSTS else registry + auth = b64encode(f"{username}:{password}".encode()).decode() + cfg = os.path.join(tmp, "config.json") + _write_private(cfg, json.dumps({"auths": {key: {"auth": auth}}})) + # docker/crane/skopeo read DOCKER_CONFIG, podman/skopeo read + # REGISTRY_AUTH_FILE, helm reads HELM_REGISTRY_CONFIG. + env["DOCKER_CONFIG"] = tmp + env["REGISTRY_AUTH_FILE"] = cfg + env["HELM_REGISTRY_CONFIG"] = cfg + + if ca_cert: + if tool in ("docker", "podman"): + print( + f"[WARN] --ca-cert is not supported for {tool}. " + "Configure the daemon's trust store for custom CAs." + ) + else: + # SSL_CERT_FILE replaces the trust store, so append the custom CA + # to the system bundle instead of losing public registries. + bundle = os.path.join(tmp, "ca-bundle.pem") + parts = [] + system = _system_ca_bundle() + if system: + with open(system) as f: + parts.append(f.read()) + with open(ca_cert) as f: + parts.append(f.read()) + _write_private(bundle, "\n".join(parts)) + env["SSL_CERT_FILE"] = bundle + + yield env + finally: + shutil.rmtree(tmp, ignore_errors=True) diff --git a/src/helm.py b/src/helm.py index a5e5954..56d3e30 100644 --- a/src/helm.py +++ b/src/helm.py @@ -6,16 +6,18 @@ import atexit import os +from auth import registry_env -def run(cmd): - result = subprocess.run(cmd, capture_output=True, text=True) + +def run(cmd, env=None): + result = subprocess.run(cmd, capture_output=True, text=True, env=env) if result.returncode != 0: cmd_str = " ".join(cmd) if isinstance(cmd, list) else cmd raise Exception(f"Command failed: {cmd_str}\n{result.stderr.strip()}") return result.stdout -def pull_chart(chart, version=None, repo_url=None, insecure=False): +def pull_chart(chart, version=None, repo_url=None, insecure=False, ca_cert=None): if os.path.isdir(chart): return chart @@ -33,6 +35,11 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): atexit.register(shutil.rmtree, parent_tmp, True) tmp = os.path.join(parent_tmp, "chart") + with registry_env("helm", ca_cert=ca_cert) as env: + return _pull(chart, version, repo_url, insecure, tmp, env) + + +def _pull(chart, version, repo_url, insecure, tmp, env): repo_added = None if repo_url: @@ -40,8 +47,8 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): add_cmd = ["helm", "repo", "add", repo_name, repo_url] if insecure: add_cmd.append("--insecure-skip-tls-verify") - run(add_cmd) - run(["helm", "repo", "update", repo_name]) + run(add_cmd, env) + run(["helm", "repo", "update", repo_name], env) repo_added = repo_name chart = f"{repo_name}/{chart}" @@ -52,11 +59,11 @@ def pull_chart(chart, version=None, repo_url=None, insecure=False): cmd.append("--insecure-skip-tls-verify") try: - run(cmd) + run(cmd, env) finally: if repo_added: subprocess.run( - ["helm", "repo", "remove", repo_added], capture_output=True + ["helm", "repo", "remove", repo_added], capture_output=True, env=env ) dirs = os.listdir(tmp) @@ -76,7 +83,7 @@ def render_chart(chart_path, values=None): return run(cmd) -def push_chart(chart_path, target_registry, chart_target=None, insecure=False): +def push_chart(chart_path, target_registry, chart_target=None, insecure=False, auth=None): pkg_dir = tempfile.mkdtemp() try: run(["helm", "package", chart_path, "--destination", pkg_dir]) @@ -93,7 +100,8 @@ def push_chart(chart_path, target_registry, chart_target=None, insecure=False): cmd = ["helm", "push", tgz, repo] if insecure: cmd.append("--insecure-skip-tls-verify") - run(cmd) + with registry_env("helm", **(auth or {})) as env: + run(cmd, env) print(f"[INFO] Chart pushed: {os.path.basename(tgz)} -> {repo}") finally: shutil.rmtree(pkg_dir, ignore_errors=True) diff --git a/src/images.py b/src/images.py index a1d7a88..6542feb 100644 --- a/src/images.py +++ b/src/images.py @@ -2,12 +2,32 @@ import subprocess import re import os -import tempfile -import base64 -import json +import time +import fnmatch import yaml +from auth import registry_env + + +class _StringLoader(yaml.SafeLoader): + """SafeLoader that keeps numbers as strings. + + Otherwise an unquoted `tag: 1.10` is parsed as the float 1.1 and the wrong + image tag would be mirrored. + """ + + +_StringLoader.yaml_implicit_resolvers = { + ch: [(tag, rx) for tag, rx in resolvers + if tag not in ("tag:yaml.org,2002:float", "tag:yaml.org,2002:int")] + for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} + +# Image refs come from untrusted charts and end up as command arguments, so +# only accept plausible references (this also rejects values starting with "-"). +_IMAGE_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/:@+-]*$") + def _walk_for_images(node, images): """Recursively walk a parsed YAML node and collect image references. @@ -51,81 +71,38 @@ def _walk_for_images(node, images): _walk_for_images(value, images) -def _get_registry_from_image(image): - """Extract registry hostname from image reference.""" - # Handle various image formats: - # - registry/namespace/repo:tag - # - namespace/repo:tag (defaults to docker.io) - # - repo:tag (defaults to docker.io) - # - registry:port/namespace/repo:tag - if '/' not in image: - # No slash, could be just repo or repo:tag - if ':' in image and not image.startswith(('http://', 'https://')): - # Might be repo:tag - return 'docker.io' +def extract_images(rendered_yaml, tools): + return _filter_valid(_extract_images(rendered_yaml, tools)) + + +def _filter_valid(images): + valid = set() + for img in images: + if _IMAGE_REF.match(img): + valid.add(img) else: - # Just repo (unlikely but handle) - return 'docker.io' - - # Split on first '/' to get potential registry - parts = image.split('/', 1) - first_part = parts[0] - - # If first part contains a dot or colon, it's likely a registry - if '.' in first_part or ':' in first_part: - return first_part - else: - # No dot or colon in first part, so it's likely a namespace (e.g., library, bitnami) - # Default to docker.io - return 'docker.io' + print(f"[WARN] Ignoring invalid image reference: {img!r}") + return valid -def _create_docker_auth_config(username, password, registry=None): - """Create a Docker-style auth config for the given credentials. - - Returns a tuple of (temp_dir_path, env_dict) where: - - temp_dir_path: Path to temporary directory containing config.json - - env_dict: Environment variables to set (e.g., {'DOCKER_CONFIG': temp_dir_path}) - """ - if not username and not password: - return None, {} - - # Create temporary directory - temp_dir = tempfile.mkdtemp(prefix='helm-mirror-auth-') - - # Determine registry if not provided - if registry is None: - # We'll need to determine this from context - for now, require registry param - # This function will be called from context where we know the image - registry = 'docker.io' # fallback - - # Create auth string - auth = base64.b64encode(f"{username}:{password}".encode()).decode() - - # Create Docker config structure - config = { - "auths": { - registry: { - "auth": auth - } - } - } - - # Write config.json - config_path = os.path.join(temp_dir, 'config.json') - with open(config_path, 'w') as f: - json.dump(config, f, indent=2) - - # Return temp dir and environment variables - return temp_dir, {"DOCKER_CONFIG": temp_dir} +def filter_excluded(images, patterns): + """Drop images matching any fnmatch pattern (e.g. 'docker.io/bitnami/*').""" + kept = set() + for img in images: + hit = next((p for p in patterns if fnmatch.fnmatch(img, p)), None) + if hit: + print(f"[SKIP] {img} (excluded by {hit!r})") + else: + kept.add(img) + return kept -def extract_images(rendered_yaml, tools): +def _extract_images(rendered_yaml, tools): images = set() # Try YAML parsing first — handles all known Helm image reference patterns try: - docs = list(yaml.safe_load_all(rendered_yaml)) + docs = list(yaml.load_all(rendered_yaml, Loader=_StringLoader)) _walk_for_images(docs, images) print("[INFO] Using YAML parsing for image extraction") return images @@ -160,46 +137,34 @@ def extract_images(rendered_yaml, tools): return images -def _image_exists(image, tools, insecure=False, ca_cert=None, username=None, password=None): +def _run_retry(cmd, env, retries): + """Run a command, retrying transient failures with exponential backoff.""" + for attempt in range(retries + 1): + try: + return subprocess.run(cmd, check=True, env=env) + except subprocess.CalledProcessError: + if attempt == retries: + raise + delay = 2 ** (attempt + 1) + print(f"[WARN] {cmd[0]} {cmd[1]} failed, retrying in {delay}s ({attempt + 1}/{retries})") + time.sleep(delay) + + +def _image_exists(image, tools, insecure=False, auth=None): """Return True when the image already exists in the registry.""" - copy_tool = tools.get("copy") - env = os.environ.copy() - - # Handle CA certificate - if ca_cert and copy_tool in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and copy_tool in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {copy_tool}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - auth_temp_dir = None - if username or password: - # Determine registry for auth - registry = _get_registry_from_image(image) - auth_temp_dir, auth_env = _create_docker_auth_config(username, password, registry) - if auth_temp_dir: - env.update(auth_env) - - try: - if copy_tool == "crane": - cmd = ["crane", "manifest", image] - if insecure: - cmd.append("--insecure") - return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 - elif copy_tool == "skopeo": - cmd = ["skopeo", "inspect", f"docker://{image}"] - if insecure: - cmd.append("--tls-verify=false") - return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 + tool = tools.get("copy") + if tool == "crane": + cmd = ["crane", "manifest", image] + if insecure: + cmd.append("--insecure") + elif tool == "skopeo": + cmd = ["skopeo", "inspect", f"docker://{image}"] + if insecure: + cmd.append("--tls-verify=false") + else: return False - finally: - # Clean up temporary auth directory - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + with registry_env(tool, **(auth or {})) as env: + return subprocess.run(cmd, capture_output=True, env=env).returncode == 0 def mirror_images( @@ -210,28 +175,27 @@ def mirror_images( dry_run=False, save_dir=None, insecure=False, - ca_cert=None, - username=None, - password=None, + auth=None, parallel=1, skip_existing=False, + retries=0, ): succeeded = [] failed = [] def _process(img): if save_dir is not None: - tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, ca_cert, username, password) + tar_path = save_image_as_tar(img, save_dir, tools, dry_run, insecure, auth, retries) print(f"[INFO] {img} -> {tar_path}") return tar_path new = rewrite_image(img, registry, prefix) - if skip_existing and not dry_run and _image_exists(new, tools, insecure, ca_cert, username, password): + if skip_existing and not dry_run and _image_exists(new, tools, insecure, auth): print(f"[SKIP] {img} -> {new} (already exists)") return new print(f"[INFO] {img} -> {new}") if not dry_run: - copy_image(img, new, tools, insecure, ca_cert, username, password) + copy_image(img, new, tools, insecure, auth, retries) return new if parallel > 1: @@ -288,135 +252,63 @@ def _image_to_filename(image): return f"{safe}.tar" -def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, ca_cert=None, username=None, password=None): +def _warn_docker_insecure(): + print( + "[WARN] --insecure has no effect for docker at the command level. " + "Add the registry to insecure-registries in /etc/docker/daemon.json." + ) + + +def save_image_as_tar(image, output_dir, tools, dry_run=False, insecure=False, auth=None, retries=0): os.makedirs(output_dir, exist_ok=True) tar_path = os.path.join(output_dir, _image_to_filename(image)) if dry_run: return tar_path - if username or password: - # Authentication is handled via temporary Docker config - pass - - env = os.environ.copy() - auth_temp_dir = None - - # Handle CA certificate - if ca_cert and tools["copy"] in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and tools["copy"] in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {tools['copy']}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - if username or password: - # Determine registry for auth - registry = _get_registry_from_image(image) - auth_temp_dir, auth_env = _create_docker_auth_config(username, password, registry) - if auth_temp_dir: - env.update(auth_env) - - try: - if tools["copy"] == "crane": - cmd = ["crane", "pull"] - if insecure: - cmd.append("--insecure") - subprocess.run(cmd + ["--format=tarball", image, tar_path], check=True, env=env) - elif tools["copy"] == "skopeo": - src = f"docker://{image}" - dst = f"docker-archive:{tar_path}" - cmd = ["skopeo", "copy"] - if insecure: - cmd += ["--src-tls-verify=false"] - subprocess.run(cmd + [src, dst], check=True, env=env) - elif tools["copy"] == "docker": + tool = tools["copy"] + tls = ["--tls-verify=false"] if insecure else [] + with registry_env(tool, **(auth or {})) as env: + if tool == "crane": + cmd = ["crane", "pull"] + (["--insecure"] if insecure else []) + _run_retry(cmd + ["--format=tarball", image, tar_path], env, retries) + elif tool == "skopeo": + cmd = ["skopeo", "copy"] + (["--src-tls-verify=false"] if insecure else []) + _run_retry(cmd + [f"docker://{image}", f"docker-archive:{tar_path}"], env, retries) + elif tool == "docker": if insecure: - print( - "[WARN] --insecure has no effect for docker at the command level. " - "Add the registry to insecure-registries in /etc/docker/daemon.json." - ) - subprocess.run(["docker", "pull", image], check=True) - subprocess.run(["docker", "save", image, "-o", tar_path], check=True) + _warn_docker_insecure() + _run_retry(["docker", "pull", image], env, retries) + _run_retry(["docker", "save", image, "-o", tar_path], env, retries) else: - tls = ["--tls-verify=false"] if insecure else [] - subprocess.run(["podman", "pull"] + tls + [image], check=True) - subprocess.run(["podman", "save", image, "-o", tar_path], check=True) - finally: - # Clean up temporary auth directory - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + _run_retry(["podman", "pull"] + tls + [image], env, retries) + _run_retry(["podman", "save", image, "-o", tar_path], env, retries) return tar_path -def copy_image(src, dst, tools, insecure=False, ca_cert=None, username=None, password=None): - if username or password: - # Authentication is handled via temporary Docker config - pass - - env = os.environ.copy() - auth_temp_dirs = [] # Track temp dirs for cleanup - - # Handle CA certificate - if ca_cert and tools["copy"] in ("crane", "skopeo"): - env["SSL_CERT_FILE"] = ca_cert - elif ca_cert and tools["copy"] in ("docker", "podman"): - print( - f"[WARN] --ca-cert is not supported for {tools['copy']}. " - "Configure the daemon's trust store for custom CAs." - ) - - # Handle authentication - need auth for both source and destination - if username or password: - # Create auth for source image - src_registry = _get_registry_from_image(src) - src_auth_temp_dir, src_auth_env = _create_docker_auth_config(username, password, src_registry) - if src_auth_temp_dir: - auth_temp_dirs.append(src_auth_temp_dir) - env.update(src_auth_env) - - # Create auth for destination image (may be same credentials) - dst_registry = _get_registry_from_image(dst) - dst_auth_temp_dir, dst_auth_env = _create_docker_auth_config(username, password, dst_registry) - if dst_auth_temp_dir: - auth_temp_dirs.append(dst_auth_temp_dir) - env.update(dst_auth_env) - - try: - if tools["copy"] == "crane": - cmd = ["crane", "copy"] - if insecure: - cmd.append("--insecure") - subprocess.run(cmd + [src, dst], check=True, env=env) - elif tools["copy"] == "skopeo": +def copy_image(src, dst, tools, insecure=False, auth=None, retries=0): + tool = tools["copy"] + tls = ["--tls-verify=false"] if insecure else [] + with registry_env(tool, **(auth or {})) as env: + if tool == "crane": + cmd = ["crane", "copy"] + (["--insecure"] if insecure else []) + _run_retry(cmd + [src, dst], env, retries) + elif tool == "skopeo": cmd = ["skopeo", "copy"] if insecure: cmd += ["--src-tls-verify=false", "--dest-tls-verify=false"] - subprocess.run(cmd + [f"docker://{src}", f"docker://{dst}"], check=True, env=env) - elif tools["copy"] == "docker": + _run_retry(cmd + [f"docker://{src}", f"docker://{dst}"], env, retries) + elif tool == "docker": if insecure: - print( - "[WARN] --insecure has no effect for docker at the command level. " - "Add the registry to insecure-registries in /etc/docker/daemon.json." - ) - subprocess.run(["docker", "pull", src], check=True) - subprocess.run(["docker", "tag", src, dst], check=True) - subprocess.run(["docker", "push", dst], check=True) + _warn_docker_insecure() + _run_retry(["docker", "pull", src], env, retries) + _run_retry(["docker", "tag", src, dst], env, retries) + _run_retry(["docker", "push", dst], env, retries) else: - tls = ["--tls-verify=false"] if insecure else [] - subprocess.run(["podman", "pull"] + tls + [src], check=True) - subprocess.run(["podman", "tag", src, dst], check=True) - subprocess.run(["podman", "push"] + tls + [dst], check=True) - finally: - # Clean up temporary auth directories - for auth_temp_dir in auth_temp_dirs: - if auth_temp_dir and os.path.exists(auth_temp_dir): - import shutil - shutil.rmtree(auth_temp_dir, ignore_errors=True) + _run_retry(["podman", "pull"] + tls + [src], env, retries) + _run_retry(["podman", "tag", src, dst], env, retries) + _run_retry(["podman", "push"] + tls + [dst], env, retries) def write_image_list(images, path): diff --git a/src/main.py b/src/main.py index fb7d9ca..7c574c2 100644 --- a/src/main.py +++ b/src/main.py @@ -1,8 +1,11 @@ # mirror_tool/main.py import argparse +import os +import sys from helm import pull_chart, render_chart, push_chart, save_chart -from images import extract_images, mirror_images, write_image_list +from images import extract_images, filter_excluded, mirror_images, write_image_list from runtime import detect_tools +from auth import registry_host def main(): @@ -34,11 +37,23 @@ def main(): ) parser.add_argument( "--username", - help="Username for registry authentication.", + help="Registry username (or set REGISTRY_USERNAME).", ) parser.add_argument( "--password", - help="Password for registry authentication.", + help="Registry password. Visible in the process list and shell history; " + "prefer --password-stdin or REGISTRY_PASSWORD.", + ) + parser.add_argument( + "--password-stdin", + action="store_true", + help="Read the registry password from the first line of stdin.", + ) + parser.add_argument( + "--auth-registry", + metavar="HOST", + help="Registry the credentials are valid for (default: host of --target-registry). " + "Credentials are never sent to any other registry.", ) parser.add_argument( "--repo-url", @@ -74,6 +89,20 @@ def main(): metavar="N", help="Number of images to mirror concurrently (default: 1).", ) + parser.add_argument( + "--retries", + type=int, + default=2, + metavar="N", + help="Retry failed image copies N times with exponential backoff (default: 2).", + ) + parser.add_argument( + "--exclude-image", + action="append", + default=[], + metavar="PATTERN", + help="Skip images matching this glob (repeatable), e.g. 'docker.io/bitnami/*'.", + ) parser.add_argument( "--skip-existing", action="store_true", @@ -85,12 +114,35 @@ def main(): if not args.save_images and not args.target_registry: parser.error("--target-registry is required unless --save-images is used") + username = args.username or os.environ.get("REGISTRY_USERNAME") + password = os.environ.get("REGISTRY_PASSWORD") + if args.password_stdin: + password = sys.stdin.readline().rstrip("\r\n") + elif args.password is not None: + print("[WARN] --password is visible to other local users; prefer --password-stdin or REGISTRY_PASSWORD.") + password = args.password + if (username is None) != (password is None): + parser.error("registry authentication needs both a username and a password") + if args.parallel < 1 or args.retries < 0: + parser.error("--parallel must be >= 1 and --retries >= 0") + if args.ca_cert and not os.path.isfile(args.ca_cert): + parser.error(f"--ca-cert file not found: {args.ca_cert}") + + auth = {"ca_cert": args.ca_cert} + if password is not None: + auth_registry = args.auth_registry or ( + registry_host(args.target_registry) if args.target_registry else None + ) + if not auth_registry: + parser.error("--auth-registry is required when using credentials with --save-images") + auth.update(registry=auth_registry, username=username, password=password) + tools = detect_tools() - chart_path = pull_chart(args.chart, args.version, args.repo_url, args.insecure) + chart_path = pull_chart(args.chart, args.version, args.repo_url, args.insecure, args.ca_cert) rendered = render_chart(chart_path, args.values) - images = extract_images(rendered, tools) + images = filter_excluded(extract_images(rendered, tools), args.exclude_image) print(f"[INFO] Found {len(images)} images") mirrored = mirror_images( @@ -101,11 +153,10 @@ def main(): args.dry_run, save_dir=args.images_dir if args.save_images else None, insecure=args.insecure, - ca_cert=args.ca_cert, - username=args.username, - password=args.password, + auth=auth, parallel=args.parallel, skip_existing=args.skip_existing, + retries=args.retries, ) if args.image_list_file: @@ -114,7 +165,7 @@ def main(): if args.save_chart: save_chart(chart_path, args.chart_dir) elif args.push_chart: - push_chart(chart_path, args.target_registry, args.chart_target, args.insecure) + push_chart(chart_path, args.target_registry, args.chart_target, args.insecure, auth) if __name__ == "__main__": diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..2f1515f --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,44 @@ +import json +import os + +import pytest + +from auth import registry_env, registry_host + + +def test_registry_host(): + assert registry_host("reg.example.com:5000/prefix") == "reg.example.com:5000" + assert registry_host("https://reg.example.com") == "reg.example.com" + + +def test_credentials_only_for_target_and_private(): + with registry_env("crane", "reg.example.com", "u", "p") as env: + cfg = os.path.join(env["DOCKER_CONFIG"], "config.json") + assert oct(os.stat(cfg).st_mode & 0o777) == "0o600" + assert list(json.load(open(cfg))["auths"]) == ["reg.example.com"] + tmp = env["DOCKER_CONFIG"] + assert not os.path.exists(tmp) + + +def test_docker_hub_key(): + with registry_env("crane", "docker.io", "u", "p") as env: + auths = json.load(open(env["REGISTRY_AUTH_FILE"]))["auths"] + assert list(auths) == ["https://index.docker.io/v1/"] + + +def test_password_requires_username_and_registry(): + with pytest.raises(ValueError): + with registry_env("crane", "reg", None, "p"): + pass + + +def test_ca_cert_is_appended_not_replacing(tmp_path): + ca = tmp_path / "ca.pem" + ca.write_text("-----BEGIN CERTIFICATE-----\nCUSTOM\n-----END CERTIFICATE-----\n") + with registry_env("skopeo", ca_cert=str(ca)) as env: + assert "CUSTOM" in open(env["SSL_CERT_FILE"]).read() + + +def test_no_credentials_leaves_env_untouched(): + with registry_env("crane") as env: + assert env.get("DOCKER_CONFIG") == os.environ.get("DOCKER_CONFIG") diff --git a/tests/test_images.py b/tests/test_images.py new file mode 100644 index 0000000..32e83ed --- /dev/null +++ b/tests/test_images.py @@ -0,0 +1,87 @@ +import subprocess + +import pytest + +import images + + +def extract(doc): + return images.extract_images(doc, {}) + + +def test_standard_image_string(): + assert extract("image: nginx:1.27") == {"nginx:1.27"} + + +def test_nested_repository_and_numeric_tag_is_not_mangled(): + doc = "image:\n repository: quay.io/foo/bar\n tag: 1.10\n" + assert extract(doc) == {"quay.io/foo/bar:1.10"} + + +def test_sibling_repository_image_version(): + doc = "repository: nvcr.io/nvidia\nimage: gpu-operator\nversion: v1.2\n" + assert extract(doc) == {"nvcr.io/nvidia/gpu-operator:v1.2"} + + +@pytest.mark.parametrize("bad", ["--insecure", "-x", "a b", "img;rm", "$(id)"]) +def test_invalid_refs_are_dropped(bad): + assert extract(f"image: '{bad}'") == set() + + +def test_exclude_patterns(): + imgs = {"docker.io/bitnami/nginx:1", "quay.io/a/b:2"} + assert images.filter_excluded(imgs, ["docker.io/bitnami/*"]) == {"quay.io/a/b:2"} + + +@pytest.mark.parametrize( + "image,expected", + [ + ("docker.io/library/nginx:1", "reg.io/mirror/library/nginx:1"), + ("nginx:1", "reg.io/mirror/nginx:1"), + ("quay.io/a/b@sha256:abc", "reg.io/mirror/a/b@sha256:abc"), + ], +) +def test_rewrite_image(image, expected): + assert images.rewrite_image(image, "reg.io", "mirror") == expected + + +def test_tar_filename_has_no_path_separators(): + assert "/" not in images._image_to_filename("a/b/../c:1") + + +def test_run_retry_retries_then_succeeds(monkeypatch): + calls = [] + + def fake_run(cmd, check, env): + calls.append(cmd) + if len(calls) < 3: + raise subprocess.CalledProcessError(1, cmd) + + monkeypatch.setattr(images.subprocess, "run", fake_run) + monkeypatch.setattr(images.time, "sleep", lambda s: None) + images._run_retry(["crane", "copy", "a", "b"], {}, retries=2) + assert len(calls) == 3 + + +def test_run_retry_gives_up(monkeypatch): + def fake_run(cmd, check, env): + raise subprocess.CalledProcessError(1, cmd) + + monkeypatch.setattr(images.subprocess, "run", fake_run) + monkeypatch.setattr(images.time, "sleep", lambda s: None) + with pytest.raises(subprocess.CalledProcessError): + images._run_retry(["crane", "copy", "a", "b"], {}, retries=1) + + +def test_mirror_images_reports_failures_but_attempts_all(monkeypatch): + seen = [] + + def fake_copy(src, dst, tools, insecure, auth, retries): + seen.append(src) + if src == "bad:1": + raise RuntimeError("boom") + + monkeypatch.setattr(images, "copy_image", fake_copy) + with pytest.raises(RuntimeError, match="1 image"): + images.mirror_images(["ok:1", "bad:1", "ok:2"], "r.io", "", {"copy": "crane"}) + assert sorted(seen) == ["bad:1", "ok:1", "ok:2"]