diff --git a/test/e2e/README.md b/test/e2e/README.md index 1f78180a8..ab3a00a04 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -161,6 +161,42 @@ Worth knowing: running any test. Raise `CLUSTER_READY_TIMEOUT` (300 seconds by default) if a large topology needs longer. +## Provisioning a VM again + +Provisioning leaves a mark once it has run to the end: `.provisioned` in the +output directory, and `/etc/nri-e2e-provisioned` in the VM. Runs read the first +one to decide whether to provision, and the second one to check that a VM which +came from a cached box really holds a provisioned cluster. Provisioning which was +interrupted or which failed therefore leaves the VM marked as what it is, and the +next run provisions it again instead of running tests against a VM with no +cluster. + +A VM which turns out not to be provisioned is provisioned, whether or not it was +asked for. That covers a VM whose provisioning was interrupted, one from an +output directory or a cached box older than the marks, and one which was never +provisioned at all: none of them can run tests, and provisioning fixes all of +them. `provision` says how much of a say the run has in that: + +| value | meaning | +| --- | --- | +| unset, `auto` | provision an existing VM if it is not provisioned (the default) | +| `1`, `yes` | provision it in any case, even if it already is | +| `no` | leave an existing VM alone, and report it if it is not provisioned | + +```shell +provision=1 ./run_tests.sh policies.test-suite ~/output-directory +``` + +A VM which does not exist yet is provisioned when it is created whatever this +says, since a VM with no cluster is of no use to any test. + +The playbook ends in `kubeadm init`, which cannot run on a VM which already has a +cluster, so a run which is about to provision an existing VM resets it first with +`kubeadm reset`. This needs the VM to be up; if it is not reachable the framework +says so and provisions the VM as it is, which fails if the earlier provisioning +had got as far as installing the cluster. Bring the VM up (`make up` in the output +directory) before provisioning it again, or start from a clean output directory. + ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index c911eb20a..0d51f0000 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -42,6 +42,23 @@ BOX_RECIPE_FILES=( files/10-bridge.conf.in ) +# Where it is recorded that a VM has been provisioned. +# +# Provisioning has to leave a mark, because nothing around it says whether it +# ever ran to the end. A Vagrantfile appears before the VM is even created, and +# the provisioned flag of vagrant is cleared by the --no-provision of a VM which +# comes from a box, so a run which was interrupted or which failed while +# provisioning would otherwise look, to the next one, exactly like a run which +# got all the way through. That next run then skips provisioning and every test +# case fails on a VM which has no cluster. +# +# There are two marks. The one in the output directory is what a run reads when +# it decides whether to provision. The one in the VM travels inside the disk +# image, so a VM created from a packaged box carries it, and a box can be checked +# rather than taken at its word. +PROVISIONED_STAMP=".provisioned" +VM_PROVISIONED_STAMP="/etc/nri-e2e-provisioned" + # How long to wait for the cluster in a VM which has just booted to become # usable. Booting a large topology and starting the control plane, the CNI # plugin and the cluster DNS all happen within this. @@ -388,6 +405,183 @@ vm-package-box() { return 0 } +vm-provisioned() { + # Usage: vm-provisioned VAGRANTDIR + # + # Return success if the VM of VAGRANTDIR has been provisioned by a run which + # got all the way through, see PROVISIONED_STAMP. + [ -f "$1/$PROVISIONED_STAMP" ] +} + +vm-provisioned-from-box() { + # Usage: vm-provisioned-from-box VAGRANTDIR + # + # Return success if the VM of VAGRANTDIR was created from a packaged box + # instead of being provisioned in place. + grep -q '^box=.' "$1/$PROVISIONED_STAMP" 2>/dev/null +} + +vm-mark-provisioned() { + # Usage: vm-mark-provisioned VAGRANTDIR VMNAME [BOXNAME] + # + # Record that the VM of VAGRANTDIR is provisioned and ready to run tests. + # BOXNAME names the packaged box it was created from, if it came from one. + local vagrantdir="$1" vmname="$2" box="${3:-}" + + { + echo "# Written by vm-setup once this VM was up with everything the" + echo "# provisioning playbook installs. Remove this file to have the" + echo "# next run provision the VM again." + echo "vm=$vmname" + echo "box=$box" + echo "k8s=$k8s_release" + echo "k8scri=$k8scri" + echo "cni=$cni_plugin$cni_release" + echo "helm=$helm_release" + echo "date=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$vagrantdir/$PROVISIONED_STAMP" || + error "cannot record the provisioning of VM $vmname" +} + +vm-unmark-provisioned() { + # Usage: vm-unmark-provisioned VAGRANTDIR + # + # Forget that the VM of VAGRANTDIR is provisioned, for as long as it takes to + # provision it again. + rm -f "$1/$PROVISIONED_STAMP" +} + +vm-provisioning-mode() { + # Usage: vm-provisioning-mode + # + # Print what the provision variable asks of a VM which is already there: + # force: provision it whether it needs it or not (provision=1, provision=yes) + # never: leave it alone even if it is not provisioned (provision=no) + # auto: provision it if it turns out not to be provisioned (the default) + # + # A VM which does not exist yet is provisioned when it is created whatever + # this says: a VM with no cluster in it is of no use to any test. + case "${provision:-}" in + ""|auto) echo auto;; + no|"0"|false|off) echo never;; + *) echo force;; + esac +} + +vm-carries-provisioning-mark() { + # Usage: vm-carries-provisioning-mark + # + # Return success if the VM which is up carries the mark the playbook writes + # when it has run to the end, see VM_PROVISIONED_STAMP. + vm-command-q "[ -f $VM_PROVISIONED_STAMP ]" > /dev/null +} + +vm-verify-provisioned() { + # Usage: vm-verify-provisioned VAGRANTDIR VMNAME + # + # Make sure the VM which is up is provisioned, and provision it if it is not. + # + # A VM without the mark of the playbook is a VM whose provisioning never + # finished, one which comes from a box or an output directory older than the + # mark, or one which was never provisioned at all. None of those can run + # tests, and all of them are fixed by provisioning the VM, so do that rather + # than hand the run a VM which does not work. provision=no says to keep hands + # off, in which case there is nothing to do but report it. + local vagrantdir="$1" vmname="$2" + + if vm-carries-provisioning-mark; then + return 0 + fi + + if [ "$(vm-provisioning-mode)" == "never" ]; then + error "VM $vmname is not provisioned: $VM_PROVISIONED_STAMP is missing. +Its provisioning either never finished, or the VM comes from a cached box or an +output directory which predates this mark. provision=no asks to leave the VM +alone, so this run cannot fix it. Drop provision=no to have it provisioned, start +from a clean output directory, or add e2e_vm_cache=refresh to replace the box." + fi + + echo "VM $vmname is not provisioned, $VM_PROVISIONED_STAMP is missing." \ + "Provisioning it..." + vm-provision-now "$vagrantdir" "$vmname" + + vm-carries-provisioning-mark || + error "provisioning VM $vmname left no $VM_PROVISIONED_STAMP behind" +} + +vm-provision-now() { + # Usage: vm-provision-now VAGRANTDIR VMNAME + # + # Run the provisioning playbook on the VM of VAGRANTDIR, which is up. + local vagrantdir="$1" vmname="$2" + + # The provisioner may have been kept out of the Vagrantfile of this VM, in + # which case vagrant has nothing to run. Put it back first. + vm-set-no-provision-flag "$vagrantdir" "" + + # The playbook ends in kubeadm init, so whatever cluster the VM has now has + # to go. + vm-kubeadm-reset "$vagrantdir" + + ( cd "$vagrantdir" && + export ANSIBLE_PIPELINING=True && + export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" && + vagrant provision ${vagrant_debug:+--debug} ) || + error "failed to provision VM $vmname" +} + +vm-set-no-provision-flag() { + # Usage: vm-set-no-provision-flag VAGRANTDIR VALUE + # + # Set E2E_NO_PROVISION in the env file of VAGRANTDIR, which is what decides + # whether the Vagrantfile has the provisioner in it at all. + # + # An env file written by an earlier run carries the flag of that run, which + # may not be what this one does about provisioning. Keep it in sync, both + # ways: a vagrant up which is not ours, from make up or make ssh, should + # leave a provisioned VM alone, and provisioning needs the provisioner in + # the Vagrantfile. + local vagrantdir="$1" value="$2" + + if [ ! -f "$vagrantdir/env" ] || + grep -q "^E2E_NO_PROVISION=$value\$" "$vagrantdir/env"; then + return 0 + fi + + sed -i "s/^E2E_NO_PROVISION=.*\$/E2E_NO_PROVISION=$value/" "$vagrantdir/env" + grep -q "^E2E_NO_PROVISION=" "$vagrantdir/env" || + echo "E2E_NO_PROVISION=$value" >> "$vagrantdir/env" +} + +vm-kubeadm-reset() { + # Usage: vm-kubeadm-reset VAGRANTDIR + # + # Tear down the Kubernetes cluster of the VM of VAGRANTDIR, if it has one. + # + # Provisioning ends in "kubeadm init", which fails on a VM which already has + # a cluster: the ports are taken, the manifests are in place and etcd has + # data. So provisioning a VM again has to start by resetting whatever is + # there. A VM which is not up, or which never got as far as installing + # kubeadm, has nothing to reset. + local vagrantdir="$1" + local ids=( "$vagrantdir"/.vagrant/machines/*/*/id ) + + if [ ! -f "${ids[0]}" ]; then + # The VM has not been created yet, so there is no cluster in it either. + return 0 + fi + + echo "resetting the Kubernetes cluster of the VM before provisioning it again..." + if ! ( cd "$vagrantdir" && vagrant ssh -c "sudo sh -xc ' + command -v kubeadm > /dev/null || exit 0 + kubeadm reset --force || true + rm -rf /etc/cni/net.d /root/.kube /home/vagrant/.kube + rm -f $VM_PROVISIONED_STAMP'" ); then + echo "WARNING: could not reset the cluster of the VM." \ + "Provisioning it as it is..." >&2 + fi +} + vm-setup() { local output_dir="$1" local vmname="$2" @@ -402,36 +596,58 @@ vm-setup() { local efi_code efi_vars kind local box_name="$distro" box_file="" package_box="" use_cached_box="" local no_provision="" e2e_no_provision="" + local provisioning_mode="$(vm-provisioning-mode)" + # Decide what this run does about provisioning. + # + # Skip it for a VM which has already been provisioned, and only for such a + # VM. The mark is written once the playbook has run to the end, or once a VM + # created from a box has been checked to carry it, see PROVISIONED_STAMP. + # # Reuse an already provisioned VM if we have one for this topology and for # the versions we are about to install. Otherwise provision as usual and # keep the result for the next run. # - # This only concerns a VM which does not exist yet. An output directory - # which already has a Vagrantfile keeps the VM and the box it was created - # from, so start from a clean output directory to benefit from the cache. - if [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then + # The box cache only concerns a VM which does not exist yet. An output + # directory which already has a Vagrantfile keeps the VM and the box it was + # created from, so start from a clean output directory to benefit from the + # cache. + if [ "$provisioning_mode" == "force" ]; then + # Provisioning was asked for explicitly, so provision whatever is here. + # Until that succeeds this VM does not count as provisioned. + vm-unmark-provisioned "$vagrantdir" + elif vm-provisioned "$vagrantdir"; then + echo "VM $vmname is already provisioned, skipping provisioning..." + # Keep the provisioner out of the Vagrantfile too: --no-provision leaves + # the machine flagged as not provisioned, so the next vagrant up, whether + # it comes from the next test case or from make ssh, would run it. + no_provision="--no-provision" + e2e_no_provision=1 + # The cluster of a VM which came from a box starts up when the VM boots, + # so it may still be starting, see the wait at the end of this function. + if vm-provisioned-from-box "$vagrantdir"; then + use_cached_box=1 + fi + elif [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then box_file="$BOX_CACHE_DIR/$(vm-box-key "$vmname").box" if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then echo "using cached provisioned VM $box_file..." use_cached_box=1 box_name="$(vm-box-name "$vmname")" distro_img="file://$box_file" - # The box already has everything the playbook installs, and - # kubeadm init cannot run a second time. Keep the provisioner out - # of the Vagrantfile as well: --no-provision leaves the machine - # flagged as not provisioned, so the next vagrant up, whether it - # comes from the next test case or from make ssh, would run it. + # The box already has everything the playbook installs, and kubeadm + # init cannot run a second time. no_provision="--no-provision" e2e_no_provision=1 else package_box=1 fi elif grep -q "^IMAGE_NAME = \"$BOX_NAME_PREFIX/" "$vagrantdir/Vagrantfile" 2>/dev/null; then - # The VM of this output directory was created from a packaged box, so it - # is provisioned whatever this run was asked to do. Skip provisioning - # here too: vagrant would otherwise run it either on a VM which is - # already up, or on one which it is re-importing from that box. + # The VM of this output directory was created from a packaged box, so its + # disk is provisioned however the run which created it ended: a box only + # exists because provisioning succeeded before it was packaged. Skip + # provisioning here too, vagrant would otherwise run it either on a VM + # which is already up, or on one which it is re-importing from that box. echo "the VM of this output directory comes from a packaged box," \ "skipping provisioning..." use_cached_box=1 @@ -547,15 +763,7 @@ vm-setup() { fi fi - # An env file written before this VM was known to come from a box has no - # flag, or a stale one. Keep it in sync, so that a vagrant up which is not - # ours, from make up or make ssh, does not provision the VM either. - if [ -n "$e2e_no_provision" ] && [ -f "$vagrantdir/env" ] && - ! grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env"; then - sed -i 's/^E2E_NO_PROVISION=.*$/E2E_NO_PROVISION=1/' "$vagrantdir/env" - grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env" || - echo "E2E_NO_PROVISION=1" >> "$vagrantdir/env" - fi + vm-set-no-provision-flag "$vagrantdir" "$e2e_no_provision" (cd "$vagrantdir"; export ANSIBLE_PIPELINING=True; @@ -567,12 +775,20 @@ vm-setup() { error "failed to vagrant init $box_name" fi - # If you want to force provisioning of already provisioned vm, + # The playbook ends in kubeadm init, which cannot run on a VM which already + # has a cluster. A VM which is about to be provisioned and which has been + # here before may well have one: an earlier run may have been interrupted + # after kubeadm init and before the end of the playbook, or this run may be + # provisioning a working VM again on purpose. Either way, reset it first. + if [ -z "$no_provision" ]; then + vm-kubeadm-reset "$vagrantdir" + fi + + # If you want to force provisioning of an already provisioned vm, # then you can set provision=1 when calling e2e test script. - # Note that this is not recommended as at least kubeinit - # cannot be called second time. But this could be used - # if the provisioning failed before kubernetes was setup. - if [ ! -z "$provision" ]; then + # This can be used if the provisioning failed before kubernetes + # was setup, or to reinstall the cluster of a VM from scratch. + if [ "$provisioning_mode" == "force" ]; then if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" vagrant provision ${vagrant_debug:+--debug} || error "failed to provision VM"); then exit 1 @@ -611,10 +827,25 @@ EOF mkdir -p "$COMMAND_OUTPUT_DIR" rm -f "$COMMAND_OUTPUT_DIR"/0* + # Record that this VM is provisioned, now that it is up and everything which + # provisions it has run. A VM which came from a box was provisioned before it + # was packaged, so look for the mark the playbook left inside it rather than + # take the box at its word. + # + # This needs vm-command, so it cannot happen before the ssh config above is + # in place. It does come before waiting for the cluster below: a VM which is + # not provisioned has no cluster to wait for, and provisioning it now beats + # timing out on a node which is never going to be ready. + if ! vm-provisioned "$vagrantdir"; then + vm-verify-provisioned "$vagrantdir" "$vmname" + vm-mark-provisioned "$vagrantdir" "$vmname" \ + "$(sed -n "s/^IMAGE_NAME = \"\($BOX_NAME_PREFIX\/.*\)\"\$/\1/p" \ + "$vagrantdir/Vagrantfile" 2>/dev/null)" + fi + # A VM which has just booted from a box, or which was brought back up after # being packaged, has a cluster which is still starting up. Wait for it - # before letting the tests run. Both waits need vm-command, so they cannot - # happen before the ssh config above is in place. + # before letting the tests run. if [ -n "$use_cached_box" ] || [ -n "$package_box" ]; then wait-for-node-ready wait-for-dns-ready diff --git a/test/e2e/playbook/provision.yaml b/test/e2e/playbook/provision.yaml index be961c0bf..2b9e930ff 100644 --- a/test/e2e/playbook/provision.yaml +++ b/test/e2e/playbook/provision.yaml @@ -563,3 +563,24 @@ ansible.builtin.systemd: name: kubelet enabled: true + + # Keep this last: it is what says that everything above succeeded. The mark + # travels with the disk image, so a VM created from a "vagrant package"d box + # carries it, and lib/vm.bash can tell a provisioned VM from one whose + # provisioning was interrupted instead of guessing, see PROVISIONED_STAMP. + - name: Record that provisioning completed + ansible.builtin.copy: + dest: /etc/nri-e2e-provisioned + mode: '0644' + content: | + # Written by test/e2e/playbook/provision.yaml when it ran to the end. + # Removing this file makes the e2e framework provision this VM again. + hostname={{ hostname }} + k8s={{ k8s_release }} + k8s_version={{ k8s_version }} + cri={{ cri_runtime }} + containerd={{ containerd_release }} + crio={{ crio_release }} + cni={{ cni_plugin }}{{ cni_release }} + helm={{ helm_release }} + date={{ ansible_date_time.iso8601 }}