diff --git a/test/e2e/README.md b/test/e2e/README.md index 37e227c02..1f78180a8 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -97,6 +97,70 @@ Before running E2E tests ensure that you have all the required components locall policies.test-suite balloons test10-health-checking : PASS ``` +## Caching a provisioned VM + +Most of the time a test run spends before the first test case goes into +provisioning the VM: installing Kubernetes, the container runtime, the CNI +plugin and Helm, and initializing a single-node cluster with `kubeadm`. The +result only depends on the versions installed, so it can be exported with +`vagrant package` and reused. + +Set `e2e_vm_cache` to opt in: + +```shell +e2e_vm_cache=yes ./run_tests.sh policies.test-suite +``` + +| value | meaning | +| --- | --- | +| `no` | do not use or create cached boxes (the default) | +| `yes` | use a cached box if there is one, otherwise provision and keep the result | +| `refresh` | ignore any cached box, provision from scratch, then replace it | +| `cleanup` | remove all cached boxes, printing each of them, and exit without running tests | +| `nuke`, `drop` | synonyms of `cleanup` | + +The boxes live in `$CACHE_DIR/boxes`, next to the tarballs the framework +already caches, and are named after everything which shapes the guest: the +topology, the distro, the Kubernetes, runtime, CNI and Helm versions, and a +hash of the files which provisioning uses, listed in `BOX_RECIPE_FILES` in +`lib/vm.bash`. Editing any of them therefore invalidates the boxes instead of a +later run reusing an image which predates the change. + +Worth knowing: + +- The topology is part of the name. The hostname of the VM is derived from it, + and `kubeadm` bakes the hostname into the name of the node, into the + certificates and into etcd, so a box only serves the topology it was made + for. + +- A box is only used when the VM is created for the first time. An output + directory which already has a `Vagrantfile` keeps the VM and the box it was + created from, so start from a clean output directory to benefit from the + cache. + +- Expect a couple of gigabytes per box, and note that vagrant unpacks a box + into `~/.vagrant.d/boxes` on first use, so the disk cost is roughly twice the + file size. Nothing prunes them automatically, so clean up when done: + + ```shell + e2e_vm_cache=cleanup ./run_tests.sh policies.test-suite + ``` + + This removes both halves of every cached box, the file and the copy vagrant + unpacked, and leaves the downloaded distro images alone. + +- Boxes expire after 30 days (`BOX_CACHE_DECAY`). They contain a cluster whose + certificates the `kubeadm` defaults give a year to live, so they are not + meant to be kept around indefinitely. + +- `vagrant package` needs a recent enough `vagrant-qemu`. With an older one the + framework says so and provisions from scratch. + +- A VM which has just booted from a box has a cluster which is still starting + up, so the framework waits for the node and for the cluster DNS before + running any test. Raise `CLUSTER_READY_TIMEOUT` (300 seconds by default) if a + large topology needs longer. + ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index 6f7b13adb..f40a314a0 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -26,6 +26,7 @@ CRIO_SRC = "" CONTAINERD_RELEASE = "" CONTAINERD_SRC = "" CACHE_DIR="#{ENV['CACHE_DIR']}" +NO_PROVISION = "#{ENV['E2E_NO_PROVISION']}" if CRI_RUNTIME == "containerd" CONTAINERD_RELEASE = "#{ENV['containerd_release']}" @@ -104,31 +105,38 @@ Vagrant.configure("2") do |config| qemu.disk_resize = "#{ENV['VM_DISK_SIZE']}" end - config.vm.provision :ansible do |ansible| - ansible.playbook = "#{ENV['nri_resource_policy_src']}/test/e2e/playbook/provision.yaml" - ansible.extra_vars = { - network: "10.217.0.0/16", - hostname: HOSTNAME, - dns_nameserver: "#{ENV['DNS_NAMESERVER']}", - dns_search_domain: "#{ENV['DNS_SEARCH_DOMAIN']}", - https_proxy: "#{ENV['HTTPS_PROXY']}", - http_proxy: "#{ENV['HTTP_PROXY']}", - no_proxy: "#{ENV['NO_PROXY']}", - kernel_getsource: KERNEL_GETSOURCE, - kernel_config: KERNEL_CONFIG, - k8s_release: K8S_RELEASE, - k8s_version: K8S_VERSION, - helm_release: HELM_RELEASE, - cri_runtime: CRI_RUNTIME, - containerd_release: CONTAINERD_RELEASE, - containerd_src: CONTAINERD_SRC, - crio_release: CRIO_RELEASE, - crio_src: CRIO_SRC, - cache_dir: CACHE_DIR, - cni_plugin: CNI_PLUGIN, - cni_release: CNI_RELEASE, - nri_resource_policy_src: NRI_RESOURCE_POLICY_SRC, - outdir: OUTPUT_DIR, - } + # A VM created from an already provisioned box must never be provisioned + # again: the playbook has already run into that box, and kubeadm init + # cannot run a second time. Skipping it on the vagrant up which creates + # the VM is not enough, as vagrant then keeps the machine flagged as not + # provisioned and runs the provisioner on the next up. + if NO_PROVISION != "1" + config.vm.provision :ansible do |ansible| + ansible.playbook = "#{ENV['nri_resource_policy_src']}/test/e2e/playbook/provision.yaml" + ansible.extra_vars = { + network: "10.217.0.0/16", + hostname: HOSTNAME, + dns_nameserver: "#{ENV['DNS_NAMESERVER']}", + dns_search_domain: "#{ENV['DNS_SEARCH_DOMAIN']}", + https_proxy: "#{ENV['HTTPS_PROXY']}", + http_proxy: "#{ENV['HTTP_PROXY']}", + no_proxy: "#{ENV['NO_PROXY']}", + kernel_getsource: KERNEL_GETSOURCE, + kernel_config: KERNEL_CONFIG, + k8s_release: K8S_RELEASE, + k8s_version: K8S_VERSION, + helm_release: HELM_RELEASE, + cri_runtime: CRI_RUNTIME, + containerd_release: CONTAINERD_RELEASE, + containerd_src: CONTAINERD_SRC, + crio_release: CRIO_RELEASE, + crio_src: CRIO_SRC, + cache_dir: CACHE_DIR, + cni_plugin: CNI_PLUGIN, + cni_release: CNI_RELEASE, + nri_resource_policy_src: NRI_RESOURCE_POLICY_SRC, + outdir: OUTPUT_DIR, + } + end end end diff --git a/test/e2e/files/env.in b/test/e2e/files/env.in index f6d37a23d..9eef28274 100644 --- a/test/e2e/files/env.in +++ b/test/e2e/files/env.in @@ -13,3 +13,4 @@ DNS_SEARCH_DOMAIN="" SSH_PORT= CACHE_DIR= +E2E_NO_PROVISION= diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 746b62251..c911eb20a 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -6,6 +6,47 @@ export VM_SAVED_PROMPT="" CACHE_DIR="${CACHE_DIR:-$HOME/.cache/nri-plugins/e2e}" CACHE_DECAY="${CACHE_DECAY:-$((3 * 24 * 3600))}" # global cached variables valid for 3 days +# Cache of "vagrant package"d images of fully provisioned VMs. Reusing one +# skips installing Kubernetes, the container runtime and the CNI plugin. +# +# Expect a box to be a couple of gigabytes, and note that vagrant unpacks it +# into ~/.vagrant.d/boxes on first use, so the disk cost of a box is roughly +# twice its file size. +# +# The boxes contain a running single-node cluster whose certificates the +# kubeadm defaults give a year to live, so they must not be kept for too long. +BOX_CACHE_DIR="${BOX_CACHE_DIR:-$CACHE_DIR/boxes}" +BOX_CACHE_DECAY="${BOX_CACHE_DECAY:-$((30 * 24 * 3600))}" # cached boxes valid for 30 days + +# Name prefix of the vagrant boxes of packaged VMs. It has to differ from the +# name of the distro box, or adding a packaged box would overwrite the distro +# image which was downloaded for it. +BOX_NAME_PREFIX="${BOX_NAME_PREFIX:-nri-e2e}" + +# Files whose content shapes a provisioned VM. The name of a cached box has a +# hash of them, so that editing any of them invalidates the cached boxes +# instead of a later run reusing an image which predates the change. +# +# The paths are relative to test/e2e. Only what provisioning itself uses +# belongs here: the playbooks which deploy a plugin and the ones which install +# a custom kernel run per test run, after a box has been packaged, so they do +# not affect what is in it. +# +# Add a file here when provisioning starts using one. +BOX_RECIPE_FILES=( + playbook/provision.yaml + files/Vagrantfile.in + files/env.in + files/containerd-nri-enable + files/crio-nri-enable + files/10-bridge.conf.in +) + +# How long to wait for the cluster in a VM which has just booted to become +# usable. Booting a large topology and starting the control plane, the CNI +# plugin and the cluster DNS all happen within this. +CLUSTER_READY_TIMEOUT="${CLUSTER_READY_TIMEOUT:-300}" + error() { (echo ""; echo "error: $1" ) >&2 exit 1 @@ -117,6 +158,236 @@ vm-load-cached-var() { return 1 } +vm-box-cache-supported() { + # Usage: vm-box-cache-supported + # + # Return success if the installed vagrant-qemu implements "vagrant + # package". Older versions declare the action but do not ship the + # middlewares it uses, so calling it fails. + local version dir + + version=$(vagrant plugin list 2>/dev/null | + sed -n 's/^vagrant-qemu (\([^,)]*\).*/\1/p' | head -n 1) + if [ -z "$version" ]; then + return 1 + fi + for dir in "$HOME"/.vagrant.d/gems/*/gems/"vagrant-qemu-$version"; do + if [ -f "$dir/lib/vagrant-qemu/action/export.rb" ]; then + return 0 + fi + done + return 1 +} + +vm-box-cache-enabled() { + # Usage: vm-box-cache-enabled + # + # Return success if the box cache is in use. Set e2e_vm_cache to + # yes: use a cached box if there is one, create one if there is not + # refresh: ignore any cached box, provision from scratch, then replace it + # cleanup: remove all cached boxes and exit, see vm-cleanup-boxes + # (nuke and drop are synonyms) + # no: do not use or create cached boxes (the default) + case "${e2e_vm_cache:-no}" in + yes|1|refresh) + ;; + *) + return 1 + ;; + esac + + if ! vm-box-cache-supported; then + echo "WARNING: e2e_vm_cache=$e2e_vm_cache, but the installed" \ + "vagrant-qemu does not support \"vagrant package\"." \ + "Provisioning the VM from scratch..." >&2 + return 1 + fi + if [ -z "$k8s_release" ] || [ -z "$distro" ]; then + echo "WARNING: e2e_vm_cache=$e2e_vm_cache, but the versions to" \ + "install are not resolved yet. Provisioning the VM from scratch..." >&2 + return 1 + fi + + # Check the recipe here, where failing aborts the run. The hash of it ends + # up in the name of a box through command substitutions, which would + # swallow the failure. + vm-check-provisioning-recipe + + return 0 +} + +vm-check-provisioning-recipe() { + # Usage: vm-check-provisioning-recipe + # + # Fail unless every file in BOX_RECIPE_FILES can be read. + # + # Hashing an incomplete recipe would name boxes which do not correspond to + # it, so a file which has been renamed, moved or removed without updating + # BOX_RECIPE_FILES has to be reported rather than quietly skipped. + # + # Call this outside a command substitution. error only exits the subshell + # of one, which is why vm-provisioning-recipe-hash cannot be the only place + # which checks. + local e2e_dir="$nri_resource_policy_src/test/e2e" + local file unreadable="" + + for file in "${BOX_RECIPE_FILES[@]}"; do + if [ ! -f "$e2e_dir/$file" ] || [ ! -r "$e2e_dir/$file" ]; then + unreadable="$unreadable $file" + fi + done + if [ -n "$unreadable" ]; then + error "cannot read provisioning recipe file(s):$unreadable" + fi +} + +vm-provisioning-recipe-hash() { + # Usage: vm-provisioning-recipe-hash + # + # Print a hash of the files in BOX_RECIPE_FILES, that is, of everything + # which shapes a provisioned VM. + local e2e_dir="$nri_resource_policy_src/test/e2e" + + vm-check-provisioning-recipe + + ( cd "$e2e_dir" && cat "${BOX_RECIPE_FILES[@]}" ) | sha256sum | cut -c1-12 +} + +vm-box-key() { + # Usage: vm-box-key VMNAME + # + # Print the cache key of the box of a fully provisioned VM. + # + # VMNAME already covers the topology, the distro and the container runtime, + # and the topology has to be part of the key: the hostname of the VM is + # derived from it, and kubeadm bakes the hostname into the name of the node, + # into the certificates and into etcd. + local vmname="$1" cri_release key + + case "$k8scri" in + crio) cri_release="$crio_release";; + *) cri_release="$containerd_release";; + esac + + key="$vmname-k8s$k8s_release-$k8scri$cri_release" + key="$key-cni$cni_plugin$cni_release-helm$helm_release" + key="$key-$(vm-provisioning-recipe-hash)" + + echo "${key//[^A-Za-z0-9._-]/-}" +} + +vm-box-name() { + # Usage: vm-box-name VMNAME + # + # Print the name of the vagrant box of the packaged VM of VMNAME. + echo "$BOX_NAME_PREFIX/$(vm-box-key "$1")" +} + +vm-box-cache-cleanup-requested() { + # Usage: vm-box-cache-cleanup-requested + # + # Return success if e2e_vm_cache asks for removing the cached boxes. + case "${e2e_vm_cache:-no}" in + cleanup|nuke|drop) + return 0 + ;; + esac + return 1 +} + +vm-cleanup-boxes() { + # Usage: vm-cleanup-boxes + # + # Remove every cached box, printing each of them as it goes. + # + # This removes both halves of a cached box: the file in the box cache, and + # the copy which vagrant unpacked into its own box directory when a VM was + # created from it. Together they take a couple of gigabytes per box. + local box name files=0 boxes=0 bytes=0 + + for box in "$BOX_CACHE_DIR"/*.box "$BOX_CACHE_DIR"/*.box.tmp.*; do + if [ ! -f "$box" ]; then + continue + fi + echo "removing $box ($(du -h "$box" | cut -f1))" + bytes=$(( bytes + $(stat -c %s "$box") )) + if rm -f "$box"; then + files=$(( files + 1 )) + else + echo "WARNING: failed to remove $box" >&2 + fi + done + + while read -r name _; do + case "$name" in + "$BOX_NAME_PREFIX"/*) + echo "removing vagrant box $name" + if vagrant box remove --force "$name" > /dev/null 2>&1; then + boxes=$(( boxes + 1 )) + else + echo "WARNING: failed to remove vagrant box $name" >&2 + fi + ;; + esac + done < <(vagrant box list 2>/dev/null) + + if [ "$files" = 0 ] && [ "$boxes" = 0 ]; then + echo "no cached boxes to remove" + else + echo "removed $files cached box file(s), $(( bytes / (1024 * 1024) )) MB" \ + "from $BOX_CACHE_DIR, and $boxes vagrant box(es)" + fi +} + +vm-cached-box-usable() { + # Usage: vm-cached-box-usable BOXFILE + # + # Return success if BOXFILE exists and is not too old to be reused. + local box="$1" + + if [ ! -f "$box" ]; then + return 1 + fi + if [ $(( $(stat -c %Y "$box") + BOX_CACHE_DECAY )) -lt $(date +%s) ]; then + echo "cached box $box is more than $(( BOX_CACHE_DECAY / 86400 )) days" \ + "old, provisioning the VM from scratch..." >&2 + return 1 + fi + return 0 +} + +vm-package-box() { + # Usage: vm-package-box VAGRANTDIR BOXFILE + # + # Export the provisioned VM in VAGRANTDIR into BOXFILE. + # + # Note that packaging shuts the VM down, so the caller has to bring it back + # up. Write to a temporary file first, so that a run which fails or is + # interrupted halfway does not leave a truncated box behind for the next + # one to use. + local vagrantdir="$1" box="$2" tmp="$2.tmp.$$" + + if ! mkdir -p "$(dirname "$box")"; then + echo "WARNING: cannot create box cache dir $(dirname "$box")" >&2 + return 1 + fi + + echo "packaging the provisioned VM into $box..." + if ! ( cd "$vagrantdir" && vagrant package --output "$tmp" ); then + rm -f "$tmp" + echo "WARNING: failed to package the VM into a box" >&2 + return 1 + fi + if ! mv "$tmp" "$box"; then + rm -f "$tmp" + echo "WARNING: failed to move the packaged box to $box" >&2 + return 1 + fi + + echo "packaged the provisioned VM into $box ($(du -h "$box" | cut -f1))" + return 0 +} + vm-setup() { local output_dir="$1" local vmname="$2" @@ -127,9 +398,48 @@ vm-setup() { local inventory="$playbook/inventory" local vagrantdir="$output_dir" local files="$nri_resource_policy_src/test/e2e/files" - local distro_name=$(printf '%s\n' "$distro" | sed -e 's/[\/&]/\\&/g') local qemu_dir="${qemu_dir:-/usr/share/qemu}" local efi_code efi_vars kind + local box_name="$distro" box_file="" package_box="" use_cached_box="" + local no_provision="" e2e_no_provision="" + + # Reuse an already provisioned VM if we have one for this topology and for + # the versions we are about to install. Otherwise provision as usual and + # keep the result for the next run. + # + # This only concerns a VM which does not exist yet. An output directory + # which already has a Vagrantfile keeps the VM and the box it was created + # from, so start from a clean output directory to benefit from the cache. + if [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then + box_file="$BOX_CACHE_DIR/$(vm-box-key "$vmname").box" + if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then + echo "using cached provisioned VM $box_file..." + use_cached_box=1 + box_name="$(vm-box-name "$vmname")" + distro_img="file://$box_file" + # The box already has everything the playbook installs, and + # kubeadm init cannot run a second time. Keep the provisioner out + # of the Vagrantfile as well: --no-provision leaves the machine + # flagged as not provisioned, so the next vagrant up, whether it + # comes from the next test case or from make ssh, would run it. + no_provision="--no-provision" + e2e_no_provision=1 + else + package_box=1 + fi + elif grep -q "^IMAGE_NAME = \"$BOX_NAME_PREFIX/" "$vagrantdir/Vagrantfile" 2>/dev/null; then + # The VM of this output directory was created from a packaged box, so it + # is provisioned whatever this run was asked to do. Skip provisioning + # here too: vagrant would otherwise run it either on a VM which is + # already up, or on one which it is re-importing from that box. + echo "the VM of this output directory comes from a packaged box," \ + "skipping provisioning..." + use_cached_box=1 + no_provision="--no-provision" + e2e_no_provision=1 + fi + + local distro_name=$(printf '%s\n' "$box_name" | sed -e 's/[\/&]/\\&/g') mkdir -p "$inventory" if [ ! -f "$inventory/vagrant.ini" ]; then @@ -225,24 +535,36 @@ vm-setup() { -e "s/DNS_SEARCH_DOMAIN=\"\"/DNS_SEARCH_DOMAIN=\"$dns_search_domain\"/g" \ -e "s/SSH_PORT=/SSH_PORT=$SSH_PORT/g" \ -e "s:CACHE_DIR=:CACHE_DIR=\"$CACHE_DIR\":g" \ + -e "s:E2E_NO_PROVISION=:E2E_NO_PROVISION=$e2e_no_provision:g" \ "$files/env.in" > "$vagrantdir/env" else sed -e "s/DNS_NAMESERVER=\"\"/DNS_NAMESERVER=\"$dns_nameserver\"/g" \ -e "s/DNS_SEARCH_DOMAIN=\"\"/DNS_SEARCH_DOMAIN=\"$dns_search_domain\"/g" \ -e "s/SSH_PORT=/SSH_PORT=$SSH_PORT/g" \ -e "s:CACHE_DIR=:CACHE_DIR=\"$CACHE_DIR\":g" \ + -e "s:E2E_NO_PROVISION=:E2E_NO_PROVISION=$e2e_no_provision:g" \ "$files/env.in" > "$vagrantdir/env" fi fi + # An env file written before this VM was known to come from a box has no + # flag, or a stale one. Keep it in sync, so that a vagrant up which is not + # ours, from make up or make ssh, does not provision the VM either. + if [ -n "$e2e_no_provision" ] && [ -f "$vagrantdir/env" ] && + ! grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env"; then + sed -i 's/^E2E_NO_PROVISION=.*$/E2E_NO_PROVISION=1/' "$vagrantdir/env" + grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env" || + echo "E2E_NO_PROVISION=1" >> "$vagrantdir/env" + fi + (cd "$vagrantdir"; export ANSIBLE_PIPELINING=True; # Make sure the vagrant plugins are installed make install || error "failed to check/install vagrant plugins" if [ ! -d .vagrant ]; then - vagrant init ${vagrant_debug:+--debug} --template Vagrantfile $distro || \ - error "failed to vagrant init $distro" + vagrant init ${vagrant_debug:+--debug} --template Vagrantfile $box_name || \ + error "failed to vagrant init $box_name" fi # If you want to force provisioning of already provisioned vm, @@ -258,9 +580,22 @@ vm-setup() { fi if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" - vagrant up --provider qemu || error "failed to bring up VM"); then + vagrant up $no_provision --provider qemu || error "failed to bring up VM"); then exit 1 fi + + # Keep the freshly provisioned VM for the next run. Packaging shuts the + # VM down, so bring it back up afterwards. Failing to package is not + # fatal: the VM we have is fine, we just do not get to reuse it. + if [ -n "$package_box" ]; then + vm-package-box "$vagrantdir" "$box_file" || : + if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" + vagrant up --no-provision --provider qemu || \ + error "failed to bring the VM back up after packaging it"); then + exit 1 + fi + fi + vagrant ssh-config > .ssh-config cat >> .ssh-config <