diff --git a/.github/workflows/check-ci-config.yml b/.github/workflows/check-ci-config.yml new file mode 100644 index 0000000..b3901f0 --- /dev/null +++ b/.github/workflows/check-ci-config.yml @@ -0,0 +1,20 @@ +name: Check CI config + +on: + pull_request: + branches: [main] + +jobs: + check-ci-config: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: pip install pyyaml + + - name: Regenerate .gitlab-ci.yml + run: python3 test/generate_ci.py > .gitlab-ci.yml.generated + + - name: Check .gitlab-ci.yml is up to date + run: diff -u .gitlab-ci.yml .gitlab-ci.yml.generated diff --git a/.github/workflows/update-images.yml b/.github/workflows/update-images.yml new file mode 100644 index 0000000..f3cdd01 --- /dev/null +++ b/.github/workflows/update-images.yml @@ -0,0 +1,43 @@ +name: Update images library ref + +on: + workflow_dispatch: + schedule: + - cron: "0 12 * * 0" + +jobs: + update-and-push: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }} + GITHUB_TOKEN: ${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }} + steps: + - name: Clone + run: | + git clone --depth=1 --branch main https://github.com/$GITHUB_REPOSITORY ./src + + - name: User config + working-directory: ./src + run: | + git config user.name "schutzbot" + git config user.email "schutzbot@gmail.com" + + - name: Update Schutzfile + working-directory: ./src + run: python3 test/update-schutzfile-images + + - name: Open PR + working-directory: ./src + run: | + if git diff --exit-code; then echo "No changes"; exit 0; fi + branch="schutzfile-images-$(date -I)" + git checkout -b "${branch}" + git add Schutzfile + git commit -m "schutzfile: Update images library commit ref" + remote="https://oauth2:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}" + git push -f "${remote}" "${branch}" + gh pr create \ + --title "Update images library commit ref" \ + --body-file "github_pr_body.txt" \ + --base "main" \ + --head "${branch}" diff --git a/.gitignore b/.gitignore index b829135..0283850 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ /*.local.sh /*.local.yaml +github_pr_body.txt +_images/ +build/ +*.pyc +__pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 610f8cb..79f56e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,296 @@ ---- stages: + - init - test + - finish -no-op: +init: + stage: init + interruptible: true + tags: + - shell + script: + - schutzbot/update_github_status.sh start + +finish: + stage: finish + tags: + - shell + script: + - schutzbot/update_github_status.sh finish + +fail: + stage: finish + tags: + - shell + script: + - schutzbot/update_github_status.sh fail + - exit 1 + when: on_failure + +.base: + interruptible: true + variables: + PYTHONUNBUFFERED: '1' + before_script: + - cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null + - test/setup.sh + after_script: + - schutzbot/unregister.sh || true + +.terraform: + extends: .base + tags: + - terraform + +.terraform/openstack: + extends: .base + tags: + - terraform/openstack + +stream9-qcow2-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64 + rules: + - changes: + - Schutzfile + - qcow2-amd64/**/* + - stream9-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh stream9-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 x86_64 centos-9 + - test/boot.sh + +stream9-qcow2-aarch64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-aarch64 + rules: + - changes: + - Schutzfile + - qcow2-arm64/**/* + - stream9-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh stream9-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 aarch64 centos-9 + - test/boot.sh + +stream10-qcow2-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64 + rules: + - changes: + - Schutzfile + - qcow2-amd64/**/* + - stream10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh stream10-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 x86_64 centos-10 + - test/boot.sh + +stream10-qcow2-aarch64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-aarch64 + rules: + - changes: + - Schutzfile + - qcow2-arm64/**/* + - stream10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh stream10-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 aarch64 centos-10 + - test/boot.sh + +stream10-installer-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64-large + rules: + - changes: + - Schutzfile + - qcow2-amd64/**/* + - stream10-installer + - stream10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh stream10-installer) + - PAYLOAD_REF=$(test/get-container.sh stream10-qcow2) + - test/build.sh "$CONTAINER_REF" bootc-generic-iso x86_64 centos-10 "$PAYLOAD_REF" + - test/boot.sh + +rhel-10-azure-x86_64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-x86_64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - azure-amd64/**/* + - azure/**/* + - rhel-10-azure + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-azure) + - test/build.sh "$CONTAINER_REF" vhd x86_64 rhel-10 + - test/boot.sh + +rhel-10-ec2-x86_64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-x86_64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - ec2-amd64/**/* + - ec2/**/* + - rhel-10-ec2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-ec2) + - test/build.sh "$CONTAINER_REF" ami x86_64 rhel-10 + - test/boot.sh + +rhel-10-ec2-aarch64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-aarch64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - ec2-arm64/**/* + - ec2/**/* + - rhel-10-ec2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-ec2) + - test/build.sh "$CONTAINER_REF" ami aarch64 rhel-10 + - test/boot.sh + +rhel-10-gcp-x86_64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-x86_64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - gcp-amd64/**/* + - gcp/**/* + - rhel-10-gcp + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-gcp) + - test/build.sh "$CONTAINER_REF" gce x86_64 rhel-10 + - test/boot.sh + +rhel-10-installer-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64-large + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - qcow2-amd64/**/* + - rhel-10-installer + - rhel-10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-installer) + - PAYLOAD_REF=$(test/get-container.sh rhel-10-qcow2) + - test/build.sh "$CONTAINER_REF" bootc-generic-iso x86_64 rhel-10 "$PAYLOAD_REF" + - test/boot.sh + +rhel-10-qcow2-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - qcow2-amd64/**/* + - rhel-10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 x86_64 rhel-10 + - test/boot.sh + +rhel-10-qcow2-aarch64: + stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-aarch64 + SUBSCRIPTION_NEEDED: true + RH_REGISTRY_LOGIN_NEEDED: true + rules: + - changes: + - Schutzfile + - qcow2-arm64/**/* + - rhel-10-qcow2 + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh rhel-10-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 aarch64 rhel-10 + - test/boot.sh + +hummingbird-qcow2-x86_64: + stage: test + extends: .terraform/openstack + variables: + RUNNER: rhos-01/fedora-43-x86_64 + rules: + - changes: + - Schutzfile + - hummingbird-qcow2 + - hummingbird-qcow2-amd64/**/* + - qcow2-amd64/**/* + - test/**/* + script: + - CONTAINER_REF=$(test/get-container.sh hummingbird-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 x86_64 hummingbird + - test/boot.sh + +hummingbird-qcow2-aarch64: stage: test + extends: .terraform + variables: + RUNNER: aws/fedora-43-aarch64 + rules: + - changes: + - Schutzfile + - hummingbird-qcow2 + - hummingbird-qcow2-arm64/**/* + - qcow2-arm64/**/* + - test/**/* script: - - echo "no-op" + - CONTAINER_REF=$(test/get-container.sh hummingbird-qcow2) + - test/build.sh "$CONTAINER_REF" qcow2 aarch64 hummingbird + - test/boot.sh diff --git a/.yamllint.yaml b/.yamllint.yaml index 9f3b20a..b30dbc0 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -6,6 +6,9 @@ ignore: | rules: document-start: disable + indentation: + spaces: 2 + indent-sequences: consistent line-length: max: 200 truthy: disable diff --git a/Makefile b/Makefile index 00bad33..24511c9 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,11 @@ fmt: fmt-shell .PHONY: fmt-shell fmt-shell: find . -name '*.sh' -not -path '*/.git/*' -exec shfmt -w -i 4 {} \; + +.PHONY: test +test: + python3 -m pytest test/ -v + +.PHONY: generate-ci +generate-ci: + python3 test/generate_ci.py > .gitlab-ci.yml diff --git a/Schutzfile b/Schutzfile new file mode 100644 index 0000000..f02bd3d --- /dev/null +++ b/Schutzfile @@ -0,0 +1,9 @@ +{ + "common": { + "dependencies": { + "images": { + "ref": "c1fbb6ad0758cb1f13f5be6b6d45e50c140b7c2a" + } + } + } +} diff --git a/rhel-10-installer b/rhel-10-installer index 350714b..6dd656c 100644 --- a/rhel-10-installer +++ b/rhel-10-installer @@ -63,8 +63,8 @@ EOT COPY <> /etc/passwd && \ diff --git a/schutzbot/team_ssh_keys.txt b/schutzbot/team_ssh_keys.txt new file mode 100644 index 0000000..70b6ba0 --- /dev/null +++ b/schutzbot/team_ssh_keys.txt @@ -0,0 +1,11 @@ +# SSH keys from members of the osbuild team that are used in CI. +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPB1jFl4p6FTBixHT6wOk6X8nj/Z7eoPNQE/M0wK485K obudai@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAw6IgsAlMJQlOtJXvtlY1racZPntLiy4+iDwrPMCgbYbsylY5TI2S4JCzC3OsnOF/abozKOhTrX04KOSOPkG8iZjBEUsMX4rQXtdViyec8pAdKOimzN9tdlfC2joW8jPlr/wpKMnMRCQmNDUZIOl1ujyTeY592JE8sj9TTqyc+fk= bcl@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIAYU2wzSk9r1l3iOwsvaJXCsfQIUga3xzShZJAM1zHv akoutsou-R@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM+4pso8s0M0hKFW6XoEvM6loZp0C7D9ZlmwXQbhxyV0 akoutsou-i@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINPExjjH74MOM6wrXEpRUg6I0dtRdAV3bAUY+u7WMc2G sanne@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjNynFZPCEPVDyOB2yzrww5kxwK6MAb1D0GN5yP8y/iw+gtx+Hj3CqojHMTa/9r3q3R1TMgCITdvzAiKylbx/owV8bgXS1p8je2KirWx3o/Dy80AYsas2F+sodm5/FOz6LvcUZw2vZiVs1wp8dz7ak+pm6Xg7xa7511xO4T/HStzNUE/XSPYmC9LNJ+uVWTiCjTWlZxp1JcDVfO7k69F60u8D42e1Ty60IeNeJItX/o8FUjB/rMAAJRpjFpd/uyfPTWamjNoVzrB7chFxaemg2Nf8na6PHLAx8Gcxz2fdnnsg+M5vr6z0yVYz1cc8VOhYynQm9iISvTt6bDVEbWc2T thozza@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINDRWitNwQc/YsOSC7Reeh7x57mSzcc+4+SayHHu/NCG sdevlieg-0@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNh/u8oWHfYwr01X8G8ijSC3hPfKfLpK8MISxg2mq1O sdevlieg-1@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBCWAwAqV3weCALKWrSAAHir+oIga1TU5VL4hnjWWU2x gzuccare@redhat.com +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEhnn80ZywmjeBFFOGm+cm+5HUwm62qTVnjKlOdYFLHN lzap@redhat.com diff --git a/schutzbot/terraform b/schutzbot/terraform new file mode 100644 index 0000000..8796f25 --- /dev/null +++ b/schutzbot/terraform @@ -0,0 +1 @@ +3545c5e293c8677f8f48b9a1e4cf6131bf288c24 diff --git a/schutzbot/unregister.sh b/schutzbot/unregister.sh new file mode 100755 index 0000000..c60a1f9 --- /dev/null +++ b/schutzbot/unregister.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Colorful output. +function greenprint { + echo -e "\033[1;32m[$(date -Isecond)] ${1}\033[0m" +} +function redprint { + echo -e "\033[1;31m[$(date -Isecond)] ${1}\033[0m" +} + +if ! hash subscription-manager; then + exit 0 +fi +if ! sudo subscription-manager status; then + exit 0 +fi +if sudo subscription-manager unregister; then + greenprint "Host unregistered." + exit 0 +fi +redprint "Failed to unregister" +exit 1 diff --git a/schutzbot/update_github_status.sh b/schutzbot/update_github_status.sh new file mode 100755 index 0000000..7b4fc41 --- /dev/null +++ b/schutzbot/update_github_status.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# if a user is logged in to the runner, wait until they're done +while (( $(who -s | wc -l) > 0 )); do + echo "Waiting for user(s) to log off" + sleep 30 +done + +if [[ $1 == "start" ]]; then + GITHUB_NEW_STATE="pending" + GITHUB_NEW_DESC="I'm currently testing this commit, be patient." +elif [[ $1 == "finish" ]]; then + GITHUB_NEW_STATE="success" + GITHUB_NEW_DESC="I like this commit!" +elif [[ $1 == "update" ]]; then + if [[ $CI_JOB_STATUS == "canceled" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="Someone told me to cancel this test run." + elif [[ $CI_JOB_STATUS == "failed" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="I'm sorry, something is odd about this commit." + else + exit 0 + fi +elif [[ $1 == "fail" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="I'm sorry, something is odd about this commit." +else + echo "unknown command" + exit 1 +fi + +CONTEXT="Schutzbot on GitLab" + +# TODO: This should be amended once we start using schedule triggers +if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then + CONTEXT="$CONTEXT, RHEL-${RHEL_MAJOR:-}-nightly" +fi + +curl \ + -u "${SCHUTZBOT_LOGIN}" \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/osbuild/bootc-foundry/statuses/${CI_COMMIT_SHA}" \ + -d '{"state":"'"${GITHUB_NEW_STATE}"'", "description": "'"${GITHUB_NEW_DESC}"'", "context": "'"${CONTEXT}"'", "target_url": "'"${CI_PIPELINE_URL}"'"}' diff --git a/stream10-installer b/stream10-installer index e70089d..b446ede 100644 --- a/stream10-installer +++ b/stream10-installer @@ -21,6 +21,7 @@ RUN dnf install -y \ google-noto-sans-cjk-fonts \ xorriso \ squashfs-tools \ + fuse-overlayfs \ && dnf clean all RUN mkdir -p /boot/efi \ @@ -39,7 +40,7 @@ grub2: EOT COPY <> /etc/passwd && \ diff --git a/test/boot.sh b/test/boot.sh new file mode 100755 index 0000000..f1efd4a --- /dev/null +++ b/test/boot.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +IMAGES_DIR="${REPO_ROOT}/_images" +BUILD_DIR="${REPO_ROOT}/build" + +# Discover the build output subdirectory +BUILD_OUTPUT_DIR=$(find "${BUILD_DIR}" -mindepth 1 -maxdepth 1 -type d ! -name ".*" | head -1) + +if [ -z "${BUILD_OUTPUT_DIR}" ]; then + echo "ERROR: No build output directory found under ${BUILD_DIR}" + exit 1 +fi + +# Find the build config in the output directory +CONFIG_FILE="${BUILD_OUTPUT_DIR}/config.json" +if [ ! -f "${CONFIG_FILE}" ]; then + echo "ERROR: Build config not found at ${CONFIG_FILE}" + exit 1 +fi + +echo "Booting image from: ${BUILD_OUTPUT_DIR}" +echo "Build config: ${CONFIG_FILE}" + +# Boot the image and run validation checks. +# Run from the images directory so Go can find the module (go.mod) when +# boot-image builds cmd/check-host-config. +(cd "${IMAGES_DIR}" && ./test/scripts/boot-image "${BUILD_OUTPUT_DIR}" "${CONFIG_FILE}") diff --git a/test/build.sh b/test/build.sh new file mode 100755 index 0000000..2197a54 --- /dev/null +++ b/test/build.sh @@ -0,0 +1,101 @@ +#!/bin/bash +set -euo pipefail + +CONTAINER_IMAGE="${1:?Usage: build.sh [payload-container-image]}" +IMAGE_TYPE="${2:?Missing image-type argument}" +ARCH="${3:?Missing arch argument}" +DISTRO_ID="${4:?Missing distro-id argument}" +PAYLOAD_CONTAINER_IMAGE="${5:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +IMAGES_DIR="${REPO_ROOT}/_images" +BUILD_DIR="${REPO_ROOT}/build" +CONFIG_FILE="${BUILD_DIR}/config.json" + +mkdir -p "${BUILD_DIR}" + +# Build the config JSON. Start with the base, optionally inject the payload ref. +cat > "${CONFIG_FILE}" < "${CONFIG_FILE}" +fi + +echo "Build config:" +cat "${CONFIG_FILE}" + +echo "" +echo "Building disk image:" +echo " Container: ${CONTAINER_IMAGE}" +echo " Image type: ${IMAGE_TYPE}" +echo " Architecture: ${ARCH}" +echo " Distro ID: ${DISTRO_ID}" +if [ -n "${PAYLOAD_CONTAINER_IMAGE}" ]; then + echo " Payload container: ${PAYLOAD_CONTAINER_IMAGE}" +fi + +# Build the disk image using the images library's cmd/build. +# Run from the images directory so Go can find the module (go.mod). +# sudo is required because cmd/build uses 'podman mount' which needs root. +(cd "${IMAGES_DIR}" && sudo go run ./cmd/build \ + --bootc-ref "${CONTAINER_IMAGE}" \ + --arch "${ARCH}" \ + --type "${IMAGE_TYPE}" \ + --config "${CONFIG_FILE}" \ + --output "${BUILD_DIR}") + +# sudo go run produces root-owned output; reclaim ownership so the rest of the +# pipeline (and boot.sh) can access the artifacts without sudo. +sudo chown -R "$(id -u):$(id -g)" "${BUILD_DIR}" + +# Discover the build output directory (single subdirectory under build/) +BUILD_OUTPUT_DIR=$(find "${BUILD_DIR}" -mindepth 1 -maxdepth 1 -type d ! -name ".*" | head -1) + +if [ -z "${BUILD_OUTPUT_DIR}" ]; then + echo "ERROR: No build output directory found under ${BUILD_DIR}/" + exit 1 +fi + +echo "Build output directory: ${BUILD_OUTPUT_DIR}" + +# Try to extract the embedded kickstart from the container image. Installer +# images ship one at /usr/share/anaconda/interactive-defaults.ks with +# the payload directive. Non-installer images won't have this file, so failures +# are silently ignored. +KS_FILENAME="embedded.ks" +# shellcheck disable=SC2024 +sudo podman run --rm "${CONTAINER_IMAGE}" \ + cat /usr/share/anaconda/interactive-defaults.ks \ + > "${BUILD_OUTPUT_DIR}/${KS_FILENAME}" 2>/dev/null || true + +# Write info.json for boot-image. +cat > "${BUILD_OUTPUT_DIR}/info.json" < "${BUILD_OUTPUT_DIR}/info.json" +fi + +# Copy the build config into the output directory for boot.sh +cp "${CONFIG_FILE}" "${BUILD_OUTPUT_DIR}/config.json" + +echo "Build complete: ${BUILD_OUTPUT_DIR}" diff --git a/test/config.yml b/test/config.yml new file mode 100644 index 0000000..e887a2d --- /dev/null +++ b/test/config.yml @@ -0,0 +1,65 @@ +images: + centos-9: + runner: aws/fedora-43 + containerfiles: + - containerfile: stream9-qcow2 + runner: rhos-01/fedora-43 + image-type: qcow2 + arches: [x86_64] + - containerfile: stream9-qcow2 + image-type: qcow2 + arches: [aarch64] + + centos-10: + runner: aws/fedora-43 + containerfiles: + - containerfile: stream10-qcow2 + runner: rhos-01/fedora-43 + image-type: qcow2 + arches: [x86_64] + - containerfile: stream10-qcow2 + image-type: qcow2 + arches: [aarch64] + - containerfile: stream10-installer + runner: rhos-01/fedora-43-{arch}-large + image-type: bootc-generic-iso + payload-containerfile: stream10-qcow2 + arches: [x86_64] + + rhel-10: + runner: aws/fedora-43 + subscription-needed: true + rh-registry-login-needed: true + containerfiles: + - containerfile: rhel-10-azure + image-type: vhd + arches: [x86_64] + - containerfile: rhel-10-ec2 + image-type: ami + arches: [x86_64, aarch64] + - containerfile: rhel-10-gcp + image-type: gce + arches: [x86_64] + - containerfile: rhel-10-installer + runner: rhos-01/fedora-43-{arch}-large + image-type: bootc-generic-iso + payload-containerfile: rhel-10-qcow2 + arches: [x86_64] + - containerfile: rhel-10-qcow2 + runner: rhos-01/fedora-43 + image-type: qcow2 + arches: [x86_64] + - containerfile: rhel-10-qcow2 + image-type: qcow2 + arches: [aarch64] + + hummingbird: + runner: aws/fedora-43 + containerfiles: + - containerfile: hummingbird-qcow2 + runner: rhos-01/fedora-43 + image-type: qcow2 + arches: [x86_64] + - containerfile: hummingbird-qcow2 + image-type: qcow2 + arches: [aarch64] diff --git a/test/generate_ci.py b/test/generate_ci.py new file mode 100755 index 0000000..9f5f73e --- /dev/null +++ b/test/generate_ci.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Generate .gitlab-ci.yml from test/config.yml and Containerfile COPY directives.""" + +import re +import sys +import yaml +from pathlib import Path + +ARCH_TO_DOCKER = { + "x86_64": "amd64", + "aarch64": "arm64", +} + +COPY_RE = re.compile(r"^COPY\s+(\S+(?:\s+\S+)+)") + + +def parse_copy_sources(lines, arch, repo_root): + """Parse COPY directives and return (sorted_dir_globs, sorted_files). + + Extracts COPY sources from Containerfile lines, substitutes ${TARGETARCH} + with the Docker arch name, and validates each source against the filesystem + rooted at repo_root. + + Returns a tuple of two sorted lists: + - directory globs: '/**/*' patterns for directory sources + - files: paths for plain file sources + + Raises ValueError for COPY with options or sources that don't exist on disk. + """ + docker_arch = ARCH_TO_DOCKER[arch] + dirs = set() + files = set() + for line in lines: + stripped = line.strip() + if stripped.startswith("COPY <<"): + continue + if stripped.startswith("COPY --"): + raise ValueError(f"COPY with options is not supported: {stripped}") + m = COPY_RE.match(stripped) + if not m: + continue + tokens = m.group(1).split() + sources = tokens[:-1] + for src in sources: + src_substituted = src.replace("${TARGETARCH}", docker_arch) + resolved = repo_root / src_substituted + if resolved.is_dir(): + top_dir = Path(src_substituted).parts[0] + dirs.add(top_dir) + elif resolved.is_file(): + files.add(src_substituted) + else: + raise ValueError( + f"COPY source does not exist: {src} (resolved to {resolved})" + ) + return sorted(f"{d}/**/*" for d in dirs), sorted(files) + + +def build_change_rules(cntfile_name, cntfile_lines, arch, payload_cntfile_name, payload_cntfile_lines, repo_root): + """Build the list of paths for CI change detection rules.""" + sources = {cntfile_name, "test/**/*", "Schutzfile"} + if payload_cntfile_name: + sources.add(payload_cntfile_name) + + all_lines = list(cntfile_lines) + if payload_cntfile_lines: + all_lines.extend(payload_cntfile_lines) + + copy_dirs, copy_files = parse_copy_sources(all_lines, arch, repo_root) + sources.update(copy_dirs) + sources.update(copy_files) + + rules = sorted(sources) + return rules + + +def build_job(cntfile_name, image_type, arch, runner_name, distro_id, change_rules, payload_cntfile_name, subscription_needed, rh_registry_login_needed): + """Build a single CI job dictionary.""" + if "{arch}" in runner_name: + runner = runner_name.format(arch=arch) + else: + runner = f"{runner_name}-{arch}" + script = [f"CONTAINER_REF=$(test/get-container.sh {cntfile_name})"] + + if payload_cntfile_name: + script.append(f"PAYLOAD_REF=$(test/get-container.sh {payload_cntfile_name})") + + build_cmd = f'test/build.sh "$CONTAINER_REF" {image_type} {arch} {distro_id}' + if payload_cntfile_name: + build_cmd += ' "$PAYLOAD_REF"' + script.append(build_cmd) + + script.append("test/boot.sh") + + extends = ".terraform" + if runner_name.startswith("rhos-01/"): + extends = ".terraform/openstack" + + variables = { + "RUNNER": runner, + } + + if subscription_needed: + variables["SUBSCRIPTION_NEEDED"] = True + + if rh_registry_login_needed: + variables["RH_REGISTRY_LOGIN_NEEDED"] = True + + return { + "stage": "test", + "extends": extends, + "variables": variables, + "rules": [{"changes": change_rules}], + "script": script, + } + + +def generate_ci_config(config, cntfile_cache, repo_root): + """Generate the full CI config dictionary from test config and Containerfile contents. + + cntfile_cache: dict mapping containerfile names to their lines + (used instead of reading files, so tests can provide synthetic content). + """ + ci = { + "stages": ["init", "test", "finish"], + "init": { + "stage": "init", + "interruptible": True, + "tags": ["shell"], + "script": [ + "schutzbot/update_github_status.sh start", + ] + }, + "finish": { + "stage": "finish", + "tags": ["shell"], + "script": [ + "schutzbot/update_github_status.sh finish", + ] + }, + "fail": { + "stage": "finish", + "tags": ["shell"], + "script": [ + "schutzbot/update_github_status.sh fail", + "exit 1", + ], + "when": "on_failure" + }, + ".base": { + "interruptible": True, + "variables": { + "PYTHONUNBUFFERED": "1", + }, + "before_script": [ + "cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null", + "test/setup.sh", + ], + "after_script": [ + "schutzbot/unregister.sh || true", + ], + }, + ".terraform": { + "extends": ".base", + "tags": ["terraform"], + }, + ".terraform/openstack": { + "extends": ".base", + "tags": ["terraform/openstack"], + }, + } + + for distro_id, distro_info in config["images"].items(): + distro_runner = distro_info["runner"] + for entry in distro_info["containerfiles"]: + cf_name = entry["containerfile"] + image_type = entry["image-type"] + payload_cf_name = entry.get("payload-containerfile") + runner = entry.get("runner", distro_runner) + + cf_lines = cntfile_cache[cf_name] + payload_cf_lines = None + if payload_cf_name: + payload_cf_lines = cntfile_cache[payload_cf_name] + + for arch in entry["arches"]: + change_rules = build_change_rules( + cntfile_name=cf_name, + cntfile_lines=cf_lines, + arch=arch, + payload_cntfile_name=payload_cf_name, + payload_cntfile_lines=payload_cf_lines, + repo_root=repo_root, + ) + job = build_job( + cntfile_name=cf_name, + image_type=image_type, + arch=arch, + runner_name=runner, + distro_id=distro_id, + change_rules=change_rules, + payload_cntfile_name=payload_cf_name, + subscription_needed=distro_info.get("subscription-needed", False), + rh_registry_login_needed=distro_info.get("rh-registry-login-needed", False), + ) + job_name = f"{cf_name}-{arch}" + ci[job_name] = job + + return ci + + +def main(): + repo_root = Path(__file__).resolve().parent.parent + + config_path = repo_root / "test" / "config.yml" + with open(config_path) as f: + config = yaml.safe_load(f) + + containerfile_cache = {} + for distro_info in config["images"].values(): + for entry in distro_info["containerfiles"]: + cf_name = entry["containerfile"] + if cf_name not in containerfile_cache: + containerfile_cache[cf_name] = (repo_root / cf_name).read_text().splitlines() + payload = entry.get("payload-containerfile") + if payload and payload not in containerfile_cache: + containerfile_cache[payload] = (repo_root / payload).read_text().splitlines() + + ci = generate_ci_config(config, containerfile_cache, repo_root) + + class IndentedListDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + output = yaml.dump(ci, default_flow_style=False, sort_keys=False, Dumper=IndentedListDumper, indent=2) + output = re.sub(r"\n(?=\S)", "\n\n", output) + sys.stdout.write(output) + + +if __name__ == "__main__": + main() diff --git a/test/get-container.sh b/test/get-container.sh new file mode 100755 index 0000000..566fb1b --- /dev/null +++ b/test/get-container.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +CONTAINERFILE="${1:?Usage: get-container.sh }" + +IMAGE_NAME="quay.io/redhat-services-prod/insights-management-tenant/image-builder-bootc-foundry/${CONTAINERFILE}:latest" + +echo "Building container image: ${IMAGE_NAME}" >&2 +echo "Containerfile: ${CONTAINERFILE}" >&2 + +sudo podman build -t "${IMAGE_NAME}" -f "${CONTAINERFILE}" . >&2 + +echo "Container image built: ${IMAGE_NAME}" >&2 + +# Print the container ref for downstream scripts to capture +echo "${IMAGE_NAME}" diff --git a/test/setup.sh b/test/setup.sh new file mode 100755 index 0000000..214b8da --- /dev/null +++ b/test/setup.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +IMAGES_DIR="${REPO_ROOT}/_images" + +IMAGES_REPO_URL="${IMAGES_REPO_URL:-https://github.com/osbuild/images.git}" + +# Read the images library ref from Schutzfile +IMAGES_REF=$(jq -r '.common.dependencies.images.ref' "${REPO_ROOT}/Schutzfile") + +echo "Images library ref: ${IMAGES_REF}" +echo "Images repo URL: ${IMAGES_REPO_URL}" + +# Clone the images library at the pinned ref +rm -rf "${IMAGES_DIR}" +echo "Cloning images library at ${IMAGES_REF}" +git clone "${IMAGES_REPO_URL}" "${IMAGES_DIR}" +git -C "${IMAGES_DIR}" checkout "${IMAGES_REF}" + +# Set up the osbuild RPM repository. The images library's setup-osbuild-repo +# auto-detects the host distro from /etc/os-release and reads the matching +# per-distro or common osbuild commit from its own Schutzfile. +sudo "${IMAGES_DIR}/test/scripts/setup-osbuild-repo" + +# Install all dependencies using the images library's install script +sudo "${IMAGES_DIR}/test/scripts/install-dependencies" + +# Ensure that the system is subscribed if needed +if [ "${SUBSCRIPTION_NEEDED:-false}" = "true" ]; then + if [ -z "${V2_RHN_REGISTRATION_SCRIPT:-}" ]; then + echo "ERROR: SUBSCRIPTION_NEEDED is set but V2_RHN_REGISTRATION_SCRIPT is empty" + exit 1 + fi + + echo "Registering system with Red Hat Subscription Manager" + sudo dnf install -y subscription-manager + set +x + echo "${V2_RHN_REGISTRATION_SCRIPT}" | sudo bash +fi + +if [ "${RH_REGISTRY_LOGIN_NEEDED:-false}" = "true" ]; then + if [ -z "${RH_CREDS:-}" ]; then + echo "ERROR: RH_REGISTRY_LOGIN_NEEDED is set but RH_CREDS is empty" + exit 1 + fi + + echo "Logging in to registry.redhat.io" + set +x + RH_USER="${RH_CREDS%%:*}" + RH_PASS="${RH_CREDS#*:}" + sudo podman login -u "${RH_USER}" -p "${RH_PASS}" registry.redhat.io +fi diff --git a/test/test_generate_ci.py b/test/test_generate_ci.py new file mode 100644 index 0000000..2c55814 --- /dev/null +++ b/test/test_generate_ci.py @@ -0,0 +1,535 @@ +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from generate_ci import ( + ARCH_TO_DOCKER, + build_change_rules, + build_job, + generate_ci_config, + parse_copy_sources, +) + + +@pytest.mark.parametrize( + "arch, expected_docker", + [ + pytest.param("x86_64", "amd64", id="x86_64"), + pytest.param("aarch64", "arm64", id="aarch64"), + ], +) +def test_arch_to_docker_mapping(arch, expected_docker): + assert ARCH_TO_DOCKER[arch] == expected_docker + + +def _setup_layout(root, dirs=None, files=None): + """Create directories and empty files under root.""" + for d in dirs or []: + (root / d).mkdir(parents=True, exist_ok=True) + for f in files or []: + p = root / f + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + + +@pytest.mark.parametrize( + "case", + [ + pytest.param( + dict( + lines=["COPY azure/etc/ /etc/"], + arch="x86_64", + layout_dirs=["azure/etc"], + layout_files=[], + expected_dirs=["azure/**/*"], + expected_files=[], + ), + id="dir_simple", + ), + pytest.param( + dict( + lines=["COPY rhel-10-azure /root/Containerfile"], + arch="x86_64", + layout_dirs=[], + layout_files=["rhel-10-azure"], + expected_dirs=[], + expected_files=["rhel-10-azure"], + ), + id="file_simple", + ), + pytest.param( + dict( + lines=["COPY config/myfile.conf /etc/myfile.conf"], + arch="x86_64", + layout_dirs=[], + layout_files=["config/myfile.conf"], + expected_dirs=[], + expected_files=["config/myfile.conf"], + ), + id="file_nested", + ), + pytest.param( + dict( + lines=["COPY < /dev/null", + "test/setup.sh", + ] + assert ci[".base"]["after_script"] == [ + "schutzbot/unregister.sh || true", + ] + assert ".terraform" in ci + assert ci[".terraform"]["extends"] == ".base" + assert ci[".terraform"]["tags"] == ["terraform"] + assert ".terraform/openstack" in ci + assert ci[".terraform/openstack"]["extends"] == ".base" + assert ci[".terraform/openstack"]["tags"] == ["terraform/openstack"] + + +def test_generate_ci_config_generates_job_per_arch(tmp_path): + _setup_single_distro_layout(tmp_path) + ci = generate_ci_config(SINGLE_DISTRO_CONFIG, SINGLE_DISTRO_CF_CACHE, tmp_path) + assert "stream10-qcow2-x86_64" in ci + assert "stream10-qcow2-aarch64" in ci + + +RUNNER_OVERRIDE_CONFIG = { + "images": { + "test-distro": { + "runner": "aws/default-runner", + "containerfiles": [ + { + "containerfile": "stream10-qcow2", + "image-type": "qcow2", + "arches": ["x86_64"], + }, + { + "containerfile": "stream10-installer", + "image-type": "bootc-generic-iso", + "payload-containerfile": "stream10-qcow2", + "runner": "aws/nested-virt", + "arches": ["x86_64"], + }, + ], + }, + }, +} +RUNNER_OVERRIDE_CF_CACHE = { + "stream10-qcow2": [ + "COPY qcow2-${TARGETARCH}/usr/ /usr/", + "COPY stream10-qcow2 /root/Containerfile", + ], + "stream10-installer": [ + 'COPY <