From 6fdaf2e3432ba7e7bc1d759d86a62b64d46926dd Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 27 Aug 2026 10:36:24 +0300 Subject: [PATCH 1/4] e2e: add helpers for caching a provisioned VM as a vagrant box. Provisioning a VM installs Kubernetes, a container runtime, a CNI plugin and Helm, and initializes a single-node cluster with kubeadm. That is by far the slowest part of a test run, and it is repeated for every fresh output directory although its result only depends on the versions installed. These helpers name, find and produce a "vagrant package"d image of an already provisioned VM. Nothing uses them yet. The topology is part of the key. The hostname of the VM is derived from it, and kubeadm bakes the hostname into the name of the node, into the certificates and into etcd, so a box is only reusable for the topology it was made for. The key also has a hash of the files which provisioning uses, listed in BOX_RECIPE_FILES, so that editing any of them invalidates the boxes rather than silently reusing an image which lacks the change. Editing a test case, or one of the playbooks which deploy a plugin or install a custom kernel after a box has been packaged, does not. Every file in the list has to be readable, or naming a box fails. Renaming or moving one of them without updating the list would otherwise go unnoticed and produce boxes which do not correspond to their recipe. The check runs in vm-box-cache-enabled, outside the command substitutions through which the hash reaches the name of a box: error only exits the subshell of a substitution, so failing inside one would leave a box named after an empty hash. vagrant-qemu only implements "vagrant package" in recent versions, and older ones fail in a confusing way, declaring the action but not shipping the middlewares it uses. Detect the capability rather than requiring a version number. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/vm.bash | 196 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 746b62251..ad26f786f 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -6,6 +6,37 @@ export VM_SAVED_PROMPT="" CACHE_DIR="${CACHE_DIR:-$HOME/.cache/nri-plugins/e2e}" CACHE_DECAY="${CACHE_DECAY:-$((3 * 24 * 3600))}" # global cached variables valid for 3 days +# Cache of "vagrant package"d images of fully provisioned VMs. Reusing one +# skips installing Kubernetes, the container runtime and the CNI plugin. +# +# Expect a box to be a couple of gigabytes, and note that vagrant unpacks it +# into ~/.vagrant.d/boxes on first use, so the disk cost of a box is roughly +# twice its file size. +# +# The boxes contain a running single-node cluster whose certificates the +# kubeadm defaults give a year to live, so they must not be kept for too long. +BOX_CACHE_DIR="${BOX_CACHE_DIR:-$CACHE_DIR/boxes}" +BOX_CACHE_DECAY="${BOX_CACHE_DECAY:-$((30 * 24 * 3600))}" # cached boxes valid for 30 days + +# Files whose content shapes a provisioned VM. The name of a cached box has a +# hash of them, so that editing any of them invalidates the cached boxes +# instead of a later run reusing an image which predates the change. +# +# The paths are relative to test/e2e. Only what provisioning itself uses +# belongs here: the playbooks which deploy a plugin and the ones which install +# a custom kernel run per test run, after a box has been packaged, so they do +# not affect what is in it. +# +# Add a file here when provisioning starts using one. +BOX_RECIPE_FILES=( + playbook/provision.yaml + files/Vagrantfile.in + files/env.in + files/containerd-nri-enable + files/crio-nri-enable + files/10-bridge.conf.in +) + error() { (echo ""; echo "error: $1" ) >&2 exit 1 @@ -117,6 +148,171 @@ vm-load-cached-var() { return 1 } +vm-box-cache-supported() { + # Usage: vm-box-cache-supported + # + # Return success if the installed vagrant-qemu implements "vagrant + # package". Older versions declare the action but do not ship the + # middlewares it uses, so calling it fails. + local version dir + + version=$(vagrant plugin list 2>/dev/null | + sed -n 's/^vagrant-qemu (\([^,)]*\).*/\1/p' | head -n 1) + if [ -z "$version" ]; then + return 1 + fi + for dir in "$HOME"/.vagrant.d/gems/*/gems/"vagrant-qemu-$version"; do + if [ -f "$dir/lib/vagrant-qemu/action/export.rb" ]; then + return 0 + fi + done + return 1 +} + +vm-box-cache-enabled() { + # Usage: vm-box-cache-enabled + # + # Return success if the box cache is in use. Set e2e_vm_cache to + # yes: use a cached box if there is one, create one if there is not + # refresh: ignore any cached box, provision from scratch, then replace it + # no: do not use or create cached boxes (the default) + case "${e2e_vm_cache:-no}" in + yes|1|refresh) + ;; + *) + return 1 + ;; + esac + + if ! vm-box-cache-supported; then + echo "WARNING: e2e_vm_cache=$e2e_vm_cache, but the installed" \ + "vagrant-qemu does not support \"vagrant package\"." \ + "Provisioning the VM from scratch..." >&2 + return 1 + fi + if [ -z "$k8s_release" ] || [ -z "$distro" ]; then + echo "WARNING: e2e_vm_cache=$e2e_vm_cache, but the versions to" \ + "install are not resolved yet. Provisioning the VM from scratch..." >&2 + return 1 + fi + + # Check the recipe here, where failing aborts the run. The hash of it ends + # up in the name of a box through command substitutions, which would + # swallow the failure. + vm-check-provisioning-recipe + + return 0 +} + +vm-check-provisioning-recipe() { + # Usage: vm-check-provisioning-recipe + # + # Fail unless every file in BOX_RECIPE_FILES can be read. + # + # Hashing an incomplete recipe would name boxes which do not correspond to + # it, so a file which has been renamed, moved or removed without updating + # BOX_RECIPE_FILES has to be reported rather than quietly skipped. + # + # Call this outside a command substitution. error only exits the subshell + # of one, which is why vm-provisioning-recipe-hash cannot be the only place + # which checks. + local e2e_dir="$nri_resource_policy_src/test/e2e" + local file unreadable="" + + for file in "${BOX_RECIPE_FILES[@]}"; do + if [ ! -f "$e2e_dir/$file" ] || [ ! -r "$e2e_dir/$file" ]; then + unreadable="$unreadable $file" + fi + done + if [ -n "$unreadable" ]; then + error "cannot read provisioning recipe file(s):$unreadable" + fi +} + +vm-provisioning-recipe-hash() { + # Usage: vm-provisioning-recipe-hash + # + # Print a hash of the files in BOX_RECIPE_FILES, that is, of everything + # which shapes a provisioned VM. + local e2e_dir="$nri_resource_policy_src/test/e2e" + + vm-check-provisioning-recipe + + ( cd "$e2e_dir" && cat "${BOX_RECIPE_FILES[@]}" ) | sha256sum | cut -c1-12 +} + +vm-box-key() { + # Usage: vm-box-key VMNAME + # + # Print the cache key of the box of a fully provisioned VM. + # + # VMNAME already covers the topology, the distro and the container runtime, + # and the topology has to be part of the key: the hostname of the VM is + # derived from it, and kubeadm bakes the hostname into the name of the node, + # into the certificates and into etcd. + local vmname="$1" cri_release key + + case "$k8scri" in + crio) cri_release="$crio_release";; + *) cri_release="$containerd_release";; + esac + + key="$vmname-k8s$k8s_release-$k8scri$cri_release" + key="$key-cni$cni_plugin$cni_release-helm$helm_release" + key="$key-$(vm-provisioning-recipe-hash)" + + echo "${key//[^A-Za-z0-9._-]/-}" +} + +vm-cached-box-usable() { + # Usage: vm-cached-box-usable BOXFILE + # + # Return success if BOXFILE exists and is not too old to be reused. + local box="$1" + + if [ ! -f "$box" ]; then + return 1 + fi + if [ $(( $(stat -c %Y "$box") + BOX_CACHE_DECAY )) -lt $(date +%s) ]; then + echo "cached box $box is more than $(( BOX_CACHE_DECAY / 86400 )) days" \ + "old, provisioning the VM from scratch..." >&2 + return 1 + fi + return 0 +} + +vm-package-box() { + # Usage: vm-package-box VAGRANTDIR BOXFILE + # + # Export the provisioned VM in VAGRANTDIR into BOXFILE. + # + # Note that packaging shuts the VM down, so the caller has to bring it back + # up. Write to a temporary file first, so that a run which fails or is + # interrupted halfway does not leave a truncated box behind for the next + # one to use. + local vagrantdir="$1" box="$2" tmp="$2.tmp.$$" + + if ! mkdir -p "$(dirname "$box")"; then + echo "WARNING: cannot create box cache dir $(dirname "$box")" >&2 + return 1 + fi + + echo "packaging the provisioned VM into $box..." + if ! ( cd "$vagrantdir" && vagrant package --output "$tmp" ); then + rm -f "$tmp" + echo "WARNING: failed to package the VM into a box" >&2 + return 1 + fi + if ! mv "$tmp" "$box"; then + rm -f "$tmp" + echo "WARNING: failed to move the packaged box to $box" >&2 + return 1 + fi + + echo "packaged the provisioned VM into $box ($(du -h "$box" | cut -f1))" + return 0 +} + vm-setup() { local output_dir="$1" local vmname="$2" From 82465438cc6b603cf3d4836a30819aaa049f2704 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 17:58:50 +0300 Subject: [PATCH 2/4] e2e: use a cached provisioned VM if e2e_vm_cache asks for it. Set e2e_vm_cache=yes to reuse a "vagrant package"d image of an already provisioned VM, refresh to replace one, and no, the default, to keep provisioning from scratch as before. It is off by default because the image contains a live single-node cluster which has to survive being shut down, packaged and booted again on a possibly different host. When a box is used, the VM comes up with --no-provision. The playbook has already run into that box, and kubeadm init cannot run a second time. The only thing the playbook leaves on the host is the file with the kubeadm join command, which nothing reads. When a box is created, packaging shuts the VM down, so bring it back up afterwards. Failing to package is only a warning: the VM which was just provisioned is perfectly usable, the run just does not leave a box behind for the next one. The box replaces the distro box, so it also gets a name of its own. Reusing the name of the distro box would overwrite the downloaded distro image in ~/.vagrant.d/boxes. Also, we need to prevent provision for a VM which came from a cached box. Skipping provisioning with --no-provision on the vagrant up which creates the VM is not enough. Vagrant then keeps the machine flagged as not provisioned, and runs the provisioner on the next vagrant up. That next up comes from the vm-setup of the following test case, which does not pass --no-provision: run.sh runs once per test case, and vm-setup only decides about boxes when the output directory has no Vagrantfile yet. So keep the provisioner out of the Vagrantfile instead. The Vagrantfile is generated once per output directory, which is exactly the lifetime of the decision, and this also covers a vagrant up by hand or a make ssh, both of which would otherwise re-run the playbook and break the cluster. Finally, to avoid a comparable provisioning error, never provision a VM which an output directory got from a box. Recognize a VM which came from a packaged box by the box the Vagrantfile names, and never provision it, whatever this run was asked to do and whether the VM is already up or about to be re-imported. Keep the flag of an existing env file in sync as well. An env file written before the VM was known to come from a box has no flag, so a vagrant up from make up or make ssh would still provision it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 50 ++++++++++++++++++++++++ test/e2e/files/Vagrantfile.in | 60 ++++++++++++++++------------- test/e2e/files/env.in | 1 + test/e2e/lib/vm.bash | 71 +++++++++++++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 30 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 37e227c02..c9035b348 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -97,6 +97,56 @@ Before running E2E tests ensure that you have all the required components locall policies.test-suite balloons test10-health-checking : PASS ``` +## Caching a provisioned VM + +Most of the time a test run spends before the first test case goes into +provisioning the VM: installing Kubernetes, the container runtime, the CNI +plugin and Helm, and initializing a single-node cluster with `kubeadm`. The +result only depends on the versions installed, so it can be exported with +`vagrant package` and reused. + +Set `e2e_vm_cache` to opt in: + +```shell +e2e_vm_cache=yes ./run_tests.sh policies.test-suite +``` + +| value | meaning | +| --- | --- | +| `no` | do not use or create cached boxes (the default) | +| `yes` | use a cached box if there is one, otherwise provision and keep the result | +| `refresh` | ignore any cached box, provision from scratch, then replace it | + +The boxes live in `$CACHE_DIR/boxes`, next to the tarballs the framework +already caches, and are named after everything which shapes the guest: the +topology, the distro, the Kubernetes, runtime, CNI and Helm versions, and a +hash of the files which provisioning uses, listed in `BOX_RECIPE_FILES` in +`lib/vm.bash`. Editing any of them therefore invalidates the boxes instead of a +later run reusing an image which predates the change. + +Worth knowing: + +- The topology is part of the name. The hostname of the VM is derived from it, + and `kubeadm` bakes the hostname into the name of the node, into the + certificates and into etcd, so a box only serves the topology it was made + for. + +- A box is only used when the VM is created for the first time. An output + directory which already has a `Vagrantfile` keeps the VM and the box it was + created from, so start from a clean output directory to benefit from the + cache. + +- Expect a couple of gigabytes per box, and note that vagrant unpacks a box + into `~/.vagrant.d/boxes` on first use, so the disk cost is roughly twice the + file size. Nothing prunes them automatically. + +- Boxes expire after 30 days (`BOX_CACHE_DECAY`). They contain a cluster whose + certificates the `kubeadm` defaults give a year to live, so they are not + meant to be kept around indefinitely. + +- `vagrant package` needs a recent enough `vagrant-qemu`. With an older one the + framework says so and provisions from scratch. + ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index 6f7b13adb..f40a314a0 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -26,6 +26,7 @@ CRIO_SRC = "" CONTAINERD_RELEASE = "" CONTAINERD_SRC = "" CACHE_DIR="#{ENV['CACHE_DIR']}" +NO_PROVISION = "#{ENV['E2E_NO_PROVISION']}" if CRI_RUNTIME == "containerd" CONTAINERD_RELEASE = "#{ENV['containerd_release']}" @@ -104,31 +105,38 @@ Vagrant.configure("2") do |config| qemu.disk_resize = "#{ENV['VM_DISK_SIZE']}" end - config.vm.provision :ansible do |ansible| - ansible.playbook = "#{ENV['nri_resource_policy_src']}/test/e2e/playbook/provision.yaml" - ansible.extra_vars = { - network: "10.217.0.0/16", - hostname: HOSTNAME, - dns_nameserver: "#{ENV['DNS_NAMESERVER']}", - dns_search_domain: "#{ENV['DNS_SEARCH_DOMAIN']}", - https_proxy: "#{ENV['HTTPS_PROXY']}", - http_proxy: "#{ENV['HTTP_PROXY']}", - no_proxy: "#{ENV['NO_PROXY']}", - kernel_getsource: KERNEL_GETSOURCE, - kernel_config: KERNEL_CONFIG, - k8s_release: K8S_RELEASE, - k8s_version: K8S_VERSION, - helm_release: HELM_RELEASE, - cri_runtime: CRI_RUNTIME, - containerd_release: CONTAINERD_RELEASE, - containerd_src: CONTAINERD_SRC, - crio_release: CRIO_RELEASE, - crio_src: CRIO_SRC, - cache_dir: CACHE_DIR, - cni_plugin: CNI_PLUGIN, - cni_release: CNI_RELEASE, - nri_resource_policy_src: NRI_RESOURCE_POLICY_SRC, - outdir: OUTPUT_DIR, - } + # A VM created from an already provisioned box must never be provisioned + # again: the playbook has already run into that box, and kubeadm init + # cannot run a second time. Skipping it on the vagrant up which creates + # the VM is not enough, as vagrant then keeps the machine flagged as not + # provisioned and runs the provisioner on the next up. + if NO_PROVISION != "1" + config.vm.provision :ansible do |ansible| + ansible.playbook = "#{ENV['nri_resource_policy_src']}/test/e2e/playbook/provision.yaml" + ansible.extra_vars = { + network: "10.217.0.0/16", + hostname: HOSTNAME, + dns_nameserver: "#{ENV['DNS_NAMESERVER']}", + dns_search_domain: "#{ENV['DNS_SEARCH_DOMAIN']}", + https_proxy: "#{ENV['HTTPS_PROXY']}", + http_proxy: "#{ENV['HTTP_PROXY']}", + no_proxy: "#{ENV['NO_PROXY']}", + kernel_getsource: KERNEL_GETSOURCE, + kernel_config: KERNEL_CONFIG, + k8s_release: K8S_RELEASE, + k8s_version: K8S_VERSION, + helm_release: HELM_RELEASE, + cri_runtime: CRI_RUNTIME, + containerd_release: CONTAINERD_RELEASE, + containerd_src: CONTAINERD_SRC, + crio_release: CRIO_RELEASE, + crio_src: CRIO_SRC, + cache_dir: CACHE_DIR, + cni_plugin: CNI_PLUGIN, + cni_release: CNI_RELEASE, + nri_resource_policy_src: NRI_RESOURCE_POLICY_SRC, + outdir: OUTPUT_DIR, + } + end end end diff --git a/test/e2e/files/env.in b/test/e2e/files/env.in index f6d37a23d..9eef28274 100644 --- a/test/e2e/files/env.in +++ b/test/e2e/files/env.in @@ -13,3 +13,4 @@ DNS_SEARCH_DOMAIN="" SSH_PORT= CACHE_DIR= +E2E_NO_PROVISION= diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index ad26f786f..27c6ddbc8 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -323,9 +323,47 @@ vm-setup() { local inventory="$playbook/inventory" local vagrantdir="$output_dir" local files="$nri_resource_policy_src/test/e2e/files" - local distro_name=$(printf '%s\n' "$distro" | sed -e 's/[\/&]/\\&/g') local qemu_dir="${qemu_dir:-/usr/share/qemu}" local efi_code efi_vars kind + local box_name="$distro" box_file="" package_box="" + local no_provision="" e2e_no_provision="" + + # Reuse an already provisioned VM if we have one for this topology and for + # the versions we are about to install. Otherwise provision as usual and + # keep the result for the next run. + # + # This only concerns a VM which does not exist yet. An output directory + # which already has a Vagrantfile keeps the VM and the box it was created + # from, so start from a clean output directory to benefit from the cache. + if [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then + box_file="$BOX_CACHE_DIR/$(vm-box-key "$vmname").box" + if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then + echo "using cached provisioned VM $box_file..." + box_name="nri-e2e/$(vm-box-key "$vmname")" + distro_img="file://$box_file" + # The box already has everything the playbook installs, and + # kubeadm init cannot run a second time. Keep the provisioner out + # of the Vagrantfile as well: --no-provision leaves the machine + # flagged as not provisioned, so the next vagrant up, whether it + # comes from the next test case or from make ssh, would run it. + no_provision="--no-provision" + e2e_no_provision=1 + else + package_box=1 + fi + elif grep -q "^IMAGE_NAME = \"$BOX_NAME_PREFIX/" "$vagrantdir/Vagrantfile" 2>/dev/null; then + # The VM of this output directory was created from a packaged box, so it + # is provisioned whatever this run was asked to do. Skip provisioning + # here too: vagrant would otherwise run it either on a VM which is + # already up, or on one which it is re-importing from that box. + echo "the VM of this output directory comes from a packaged box," \ + "skipping provisioning..." + use_cached_box=1 + no_provision="--no-provision" + e2e_no_provision=1 + fi + + local distro_name=$(printf '%s\n' "$box_name" | sed -e 's/[\/&]/\\&/g') mkdir -p "$inventory" if [ ! -f "$inventory/vagrant.ini" ]; then @@ -421,24 +459,36 @@ vm-setup() { -e "s/DNS_SEARCH_DOMAIN=\"\"/DNS_SEARCH_DOMAIN=\"$dns_search_domain\"/g" \ -e "s/SSH_PORT=/SSH_PORT=$SSH_PORT/g" \ -e "s:CACHE_DIR=:CACHE_DIR=\"$CACHE_DIR\":g" \ + -e "s:E2E_NO_PROVISION=:E2E_NO_PROVISION=$e2e_no_provision:g" \ "$files/env.in" > "$vagrantdir/env" else sed -e "s/DNS_NAMESERVER=\"\"/DNS_NAMESERVER=\"$dns_nameserver\"/g" \ -e "s/DNS_SEARCH_DOMAIN=\"\"/DNS_SEARCH_DOMAIN=\"$dns_search_domain\"/g" \ -e "s/SSH_PORT=/SSH_PORT=$SSH_PORT/g" \ -e "s:CACHE_DIR=:CACHE_DIR=\"$CACHE_DIR\":g" \ + -e "s:E2E_NO_PROVISION=:E2E_NO_PROVISION=$e2e_no_provision:g" \ "$files/env.in" > "$vagrantdir/env" fi fi + # An env file written before this VM was known to come from a box has no + # flag, or a stale one. Keep it in sync, so that a vagrant up which is not + # ours, from make up or make ssh, does not provision the VM either. + if [ -n "$e2e_no_provision" ] && [ -f "$vagrantdir/env" ] && + ! grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env"; then + sed -i 's/^E2E_NO_PROVISION=.*$/E2E_NO_PROVISION=1/' "$vagrantdir/env" + grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env" || + echo "E2E_NO_PROVISION=1" >> "$vagrantdir/env" + fi + (cd "$vagrantdir"; export ANSIBLE_PIPELINING=True; # Make sure the vagrant plugins are installed make install || error "failed to check/install vagrant plugins" if [ ! -d .vagrant ]; then - vagrant init ${vagrant_debug:+--debug} --template Vagrantfile $distro || \ - error "failed to vagrant init $distro" + vagrant init ${vagrant_debug:+--debug} --template Vagrantfile $box_name || \ + error "failed to vagrant init $box_name" fi # If you want to force provisioning of already provisioned vm, @@ -454,9 +504,22 @@ vm-setup() { fi if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" - vagrant up --provider qemu || error "failed to bring up VM"); then + vagrant up $no_provision --provider qemu || error "failed to bring up VM"); then exit 1 fi + + # Keep the freshly provisioned VM for the next run. Packaging shuts the + # VM down, so bring it back up afterwards. Failing to package is not + # fatal: the VM we have is fine, we just do not get to reuse it. + if [ -n "$package_box" ]; then + vm-package-box "$vagrantdir" "$box_file" || : + if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" + vagrant up --no-provision --provider qemu || \ + error "failed to bring the VM back up after packaging it"); then + exit 1 + fi + fi + vagrant ssh-config > .ssh-config cat >> .ssh-config < Date: Wed, 26 Aug 2026 19:33:59 +0300 Subject: [PATCH 3/4] e2e: wait for the cluster DNS after booting a VM from a box. A VM which has just booted from a packaged box, or which was brought back up after being packaged, has a cluster which is still starting up. A test which creates pods before the cluster DNS is up can fail for reasons which have nothing to do with what it tests. Wait for the node to be ready first: kubectl wait fails immediately if it cannot reach the API server, rather than retrying, so waiting for the deployment alone would only report that the API server is not up yet. Both waits use vm-command, so they run at the end of vm-setup. Before the ssh configuration is written and the command output directory exists, vm-command cannot work. CLUSTER_READY_TIMEOUT gives 300 seconds for all of booting a large topology, starting the control plane and the CNI plugin, and getting the DNS deployment available. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 5 +++++ test/e2e/lib/vm.bash | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index c9035b348..2d1c1f970 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -147,6 +147,11 @@ Worth knowing: - `vagrant package` needs a recent enough `vagrant-qemu`. With an older one the framework says so and provisions from scratch. +- A VM which has just booted from a box has a cluster which is still starting + up, so the framework waits for the node and for the cluster DNS before + running any test. Raise `CLUSTER_READY_TIMEOUT` (300 seconds by default) if a + large topology needs longer. + ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 27c6ddbc8..8b4376a7c 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -37,6 +37,11 @@ BOX_RECIPE_FILES=( files/10-bridge.conf.in ) +# How long to wait for the cluster in a VM which has just booted to become +# usable. Booting a large topology and starting the control plane, the CNI +# plugin and the cluster DNS all happen within this. +CLUSTER_READY_TIMEOUT="${CLUSTER_READY_TIMEOUT:-300}" + error() { (echo ""; echo "error: $1" ) >&2 exit 1 @@ -325,7 +330,7 @@ vm-setup() { local files="$nri_resource_policy_src/test/e2e/files" local qemu_dir="${qemu_dir:-/usr/share/qemu}" local efi_code efi_vars kind - local box_name="$distro" box_file="" package_box="" + local box_name="$distro" box_file="" package_box="" use_cached_box="" local no_provision="" e2e_no_provision="" # Reuse an already provisioned VM if we have one for this topology and for @@ -339,6 +344,7 @@ vm-setup() { box_file="$BOX_CACHE_DIR/$(vm-box-key "$vmname").box" if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then echo "using cached provisioned VM $box_file..." + use_cached_box=1 box_name="nri-e2e/$(vm-box-key "$vmname")" distro_img="file://$box_file" # The box already has everything the playbook installs, and @@ -534,6 +540,15 @@ EOF mkdir -p "$COMMAND_OUTPUT_DIR" rm -f "$COMMAND_OUTPUT_DIR"/0* + + # A VM which has just booted from a box, or which was brought back up after + # being packaged, has a cluster which is still starting up. Wait for it + # before letting the tests run. Both waits need vm-command, so they cannot + # happen before the ssh config above is in place. + if [ -n "$use_cached_box" ] || [ -n "$package_box" ]; then + wait-for-node-ready + wait-for-dns-ready + fi } vm-play() { @@ -1368,6 +1383,25 @@ vm-post-reboot-runtime-check() { # script API $k8scri-reimport-image $image_type } +wait-for-dns-ready() { + # Usage: [timeout=SECS] wait-for-dns-ready + # + # Wait until the cluster DNS is available, $CLUSTER_READY_TIMEOUT seconds + # at most. + # + # The cluster of a VM which has just booted is still starting up. A test + # which creates pods before the DNS is up can fail for reasons which have + # nothing to do with what it tests, so wait for it here. + # + # Note that kubectl wait fails immediately if it cannot reach the API + # server, so wait for the node to be ready before calling this. + local timeout="${timeout:-$CLUSTER_READY_TIMEOUT}" + + vm-command "kubectl -n kube-system wait deployments/coredns \ + --for=condition=Available --timeout=${timeout}s" || + command-error "cluster DNS did not become available in ${timeout}s" +} + wait-for-node-ready() { local now=$(date +%s) local deadline=$(($now + 5 * 60)) From f6d4278f2dc098398ca70eea38a5f0b3f551c9ef Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 19:44:54 +0300 Subject: [PATCH 4/4] e2e: add e2e_vm_cache=cleanup for removing cached boxes. A cached box costs a couple of gigabytes as a file and about as much again once vagrant has unpacked it, and nothing removes either half. e2e_vm_cache={cleanup|nuke|drop} removes both halves of every cached box, printing each of them as it goes, and exits without running any test. It also picks up the temporary files which an interrupted packaging leaves in the box cache. Boxes of the distro images are left alone: only the ones whose name the framework gave them are removed. The name prefix of those boxes becomes a variable of its own, so that the cleanup and the code which names them cannot disagree about which boxes belong to the framework. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 11 ++++++- test/e2e/lib/vm.bash | 72 ++++++++++++++++++++++++++++++++++++++++++- test/e2e/run_tests.sh | 5 +++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 2d1c1f970..1f78180a8 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -116,6 +116,8 @@ e2e_vm_cache=yes ./run_tests.sh policies.test-suite | `no` | do not use or create cached boxes (the default) | | `yes` | use a cached box if there is one, otherwise provision and keep the result | | `refresh` | ignore any cached box, provision from scratch, then replace it | +| `cleanup` | remove all cached boxes, printing each of them, and exit without running tests | +| `nuke`, `drop` | synonyms of `cleanup` | The boxes live in `$CACHE_DIR/boxes`, next to the tarballs the framework already caches, and are named after everything which shapes the guest: the @@ -138,7 +140,14 @@ Worth knowing: - Expect a couple of gigabytes per box, and note that vagrant unpacks a box into `~/.vagrant.d/boxes` on first use, so the disk cost is roughly twice the - file size. Nothing prunes them automatically. + file size. Nothing prunes them automatically, so clean up when done: + + ```shell + e2e_vm_cache=cleanup ./run_tests.sh policies.test-suite + ``` + + This removes both halves of every cached box, the file and the copy vagrant + unpacked, and leaves the downloaded distro images alone. - Boxes expire after 30 days (`BOX_CACHE_DECAY`). They contain a cluster whose certificates the `kubeadm` defaults give a year to live, so they are not diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 8b4376a7c..c911eb20a 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -18,6 +18,11 @@ CACHE_DECAY="${CACHE_DECAY:-$((3 * 24 * 3600))}" # global cached variables valid BOX_CACHE_DIR="${BOX_CACHE_DIR:-$CACHE_DIR/boxes}" BOX_CACHE_DECAY="${BOX_CACHE_DECAY:-$((30 * 24 * 3600))}" # cached boxes valid for 30 days +# Name prefix of the vagrant boxes of packaged VMs. It has to differ from the +# name of the distro box, or adding a packaged box would overwrite the distro +# image which was downloaded for it. +BOX_NAME_PREFIX="${BOX_NAME_PREFIX:-nri-e2e}" + # Files whose content shapes a provisioned VM. The name of a cached box has a # hash of them, so that editing any of them invalidates the cached boxes # instead of a later run reusing an image which predates the change. @@ -180,6 +185,8 @@ vm-box-cache-enabled() { # Return success if the box cache is in use. Set e2e_vm_cache to # yes: use a cached box if there is one, create one if there is not # refresh: ignore any cached box, provision from scratch, then replace it + # cleanup: remove all cached boxes and exit, see vm-cleanup-boxes + # (nuke and drop are synonyms) # no: do not use or create cached boxes (the default) case "${e2e_vm_cache:-no}" in yes|1|refresh) @@ -269,6 +276,69 @@ vm-box-key() { echo "${key//[^A-Za-z0-9._-]/-}" } +vm-box-name() { + # Usage: vm-box-name VMNAME + # + # Print the name of the vagrant box of the packaged VM of VMNAME. + echo "$BOX_NAME_PREFIX/$(vm-box-key "$1")" +} + +vm-box-cache-cleanup-requested() { + # Usage: vm-box-cache-cleanup-requested + # + # Return success if e2e_vm_cache asks for removing the cached boxes. + case "${e2e_vm_cache:-no}" in + cleanup|nuke|drop) + return 0 + ;; + esac + return 1 +} + +vm-cleanup-boxes() { + # Usage: vm-cleanup-boxes + # + # Remove every cached box, printing each of them as it goes. + # + # This removes both halves of a cached box: the file in the box cache, and + # the copy which vagrant unpacked into its own box directory when a VM was + # created from it. Together they take a couple of gigabytes per box. + local box name files=0 boxes=0 bytes=0 + + for box in "$BOX_CACHE_DIR"/*.box "$BOX_CACHE_DIR"/*.box.tmp.*; do + if [ ! -f "$box" ]; then + continue + fi + echo "removing $box ($(du -h "$box" | cut -f1))" + bytes=$(( bytes + $(stat -c %s "$box") )) + if rm -f "$box"; then + files=$(( files + 1 )) + else + echo "WARNING: failed to remove $box" >&2 + fi + done + + while read -r name _; do + case "$name" in + "$BOX_NAME_PREFIX"/*) + echo "removing vagrant box $name" + if vagrant box remove --force "$name" > /dev/null 2>&1; then + boxes=$(( boxes + 1 )) + else + echo "WARNING: failed to remove vagrant box $name" >&2 + fi + ;; + esac + done < <(vagrant box list 2>/dev/null) + + if [ "$files" = 0 ] && [ "$boxes" = 0 ]; then + echo "no cached boxes to remove" + else + echo "removed $files cached box file(s), $(( bytes / (1024 * 1024) )) MB" \ + "from $BOX_CACHE_DIR, and $boxes vagrant box(es)" + fi +} + vm-cached-box-usable() { # Usage: vm-cached-box-usable BOXFILE # @@ -345,7 +415,7 @@ vm-setup() { if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then echo "using cached provisioned VM $box_file..." use_cached_box=1 - box_name="nri-e2e/$(vm-box-key "$vmname")" + box_name="$(vm-box-name "$vmname")" distro_img="file://$box_file" # The box already has everything the playbook installs, and # kubeadm init cannot run a second time. Keep the provisioner out diff --git a/test/e2e/run_tests.sh b/test/e2e/run_tests.sh index c36764f40..acb30f7a5 100755 --- a/test/e2e/run_tests.sh +++ b/test/e2e/run_tests.sh @@ -173,6 +173,11 @@ E2E_LIB_DIR=$(realpath "$TESTS_ROOT_DIR/../lib") source "$E2E_LIB_DIR"/vm.bash +if vm-box-cache-cleanup-requested; then + vm-cleanup-boxes + exit 0 +fi + cleanup() { rm -rf "$summary_dir" }