From 8ce41110098d2fdd76011cd13df1bbfcfe7f7ee8 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 28 Aug 2026 22:04:09 +0300 Subject: [PATCH 1/3] e2e: record that provisioning a VM succeeded. Nothing said whether provisioning ever ran to the end, so a run interrupted while provisioning looked, to the next one, like a run which had finished. That run then skipped provisioning and every test case failed on a VM with no cluster. Guessing cannot replace the fact: a Vagrantfile appears before the VM is created, and the provisioned flag of vagrant is cleared by the --no-provision of a VM which comes from a box. So leave a mark, in two places. The one in the output directory is what a run reads when it decides whether to provision. The one in the VM is written by the playbook as its last task, so it travels inside the disk image and a box packaged from the VM can be checked rather than trusted. Adding that task changes the provisioning recipe hash, which invalidates the cached boxes. That is what we want: they predate the mark. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 14 +++ test/e2e/lib/vm.bash | 169 ++++++++++++++++++++++++++----- test/e2e/playbook/provision.yaml | 21 ++++ 3 files changed, 181 insertions(+), 23 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 1f78180a8..a3b803e5f 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -161,6 +161,20 @@ Worth knowing: running any test. Raise `CLUSTER_READY_TIMEOUT` (300 seconds by default) if a large topology needs longer. +## Recording that a VM is provisioned + +Provisioning leaves a mark once it has run to the end: `.provisioned` in the +output directory, and `/etc/nri-e2e-provisioned` in the VM. Runs read the first +one to decide whether to provision, and the second one to check that a VM which +came from a cached box really holds a provisioned cluster. Provisioning which was +interrupted or which failed therefore leaves the VM marked as what it is, and the +next run provisions it again instead of running tests against a VM with no +cluster. + +A VM from an output directory or a cached box which predates the marks looks +unprovisioned, and the framework says so rather than guess. Start from a clean +output directory, or add `e2e_vm_cache=refresh` to replace the box. + ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index c911eb20a..0cfa2da73 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -42,6 +42,23 @@ BOX_RECIPE_FILES=( files/10-bridge.conf.in ) +# Where it is recorded that a VM has been provisioned. +# +# Provisioning has to leave a mark, because nothing around it says whether it +# ever ran to the end. A Vagrantfile appears before the VM is even created, and +# the provisioned flag of vagrant is cleared by the --no-provision of a VM which +# comes from a box, so a run which was interrupted or which failed while +# provisioning would otherwise look, to the next one, exactly like a run which +# got all the way through. That next run then skips provisioning and every test +# case fails on a VM which has no cluster. +# +# There are two marks. The one in the output directory is what a run reads when +# it decides whether to provision. The one in the VM travels inside the disk +# image, so a VM created from a packaged box carries it, and a box can be checked +# rather than taken at its word. +PROVISIONED_STAMP=".provisioned" +VM_PROVISIONED_STAMP="/etc/nri-e2e-provisioned" + # How long to wait for the cluster in a VM which has just booted to become # usable. Booting a large topology and starting the control plane, the CNI # plugin and the cluster DNS all happen within this. @@ -388,6 +405,73 @@ vm-package-box() { return 0 } +vm-provisioned() { + # Usage: vm-provisioned VAGRANTDIR + # + # Return success if the VM of VAGRANTDIR has been provisioned by a run which + # got all the way through, see PROVISIONED_STAMP. + [ -f "$1/$PROVISIONED_STAMP" ] +} + +vm-provisioned-from-box() { + # Usage: vm-provisioned-from-box VAGRANTDIR + # + # Return success if the VM of VAGRANTDIR was created from a packaged box + # instead of being provisioned in place. + grep -q '^box=.' "$1/$PROVISIONED_STAMP" 2>/dev/null +} + +vm-mark-provisioned() { + # Usage: vm-mark-provisioned VAGRANTDIR VMNAME [BOXNAME] + # + # Record that the VM of VAGRANTDIR is provisioned and ready to run tests. + # BOXNAME names the packaged box it was created from, if it came from one. + local vagrantdir="$1" vmname="$2" box="${3:-}" + + { + echo "# Written by vm-setup once this VM was up with everything the" + echo "# provisioning playbook installs. Remove this file to have the" + echo "# next run provision the VM again." + echo "vm=$vmname" + echo "box=$box" + echo "k8s=$k8s_release" + echo "k8scri=$k8scri" + echo "cni=$cni_plugin$cni_release" + echo "helm=$helm_release" + echo "date=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$vagrantdir/$PROVISIONED_STAMP" || + error "cannot record the provisioning of VM $vmname" +} + +vm-unmark-provisioned() { + # Usage: vm-unmark-provisioned VAGRANTDIR + # + # Forget that the VM of VAGRANTDIR is provisioned, for as long as it takes to + # provision it again. + rm -f "$1/$PROVISIONED_STAMP" +} + +vm-verify-provisioned() { + # Usage: vm-verify-provisioned VMNAME + # + # Fail unless the VM which is up has been provisioned, that is, unless the + # playbook wrote VM_PROVISIONED_STAMP into it. + # + # This is what catches a VM whose provisioning never finished, and a cached + # box which was packaged from one. + local vmname="$1" + + if vm-command-q "[ -f $VM_PROVISIONED_STAMP ]" > /dev/null; then + return 0 + fi + + error "VM $vmname does not look provisioned: $VM_PROVISIONED_STAMP is missing. +Its provisioning either never finished, or the VM comes from a cached box or an +output directory which predates this check. Start from a clean output directory, +add e2e_vm_cache=refresh to replace the cached box, or provision=1 to provision +this VM again." +} + vm-setup() { local output_dir="$1" local vmname="$2" @@ -403,35 +487,56 @@ vm-setup() { local box_name="$distro" box_file="" package_box="" use_cached_box="" local no_provision="" e2e_no_provision="" + # Decide what this run does about provisioning. + # + # Skip it for a VM which has already been provisioned, and only for such a + # VM. The mark is written once the playbook has run to the end, or once a VM + # created from a box has been checked to carry it, see PROVISIONED_STAMP. + # # Reuse an already provisioned VM if we have one for this topology and for # the versions we are about to install. Otherwise provision as usual and # keep the result for the next run. # - # This only concerns a VM which does not exist yet. An output directory - # which already has a Vagrantfile keeps the VM and the box it was created - # from, so start from a clean output directory to benefit from the cache. - if [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then + # The box cache only concerns a VM which does not exist yet. An output + # directory which already has a Vagrantfile keeps the VM and the box it was + # created from, so start from a clean output directory to benefit from the + # cache. + if [ -n "$provision" ]; then + # Provisioning was asked for explicitly, so provision whatever is here. + # Until that succeeds this VM does not count as provisioned. + vm-unmark-provisioned "$vagrantdir" + elif vm-provisioned "$vagrantdir"; then + echo "VM $vmname is already provisioned, skipping provisioning..." + # Keep the provisioner out of the Vagrantfile too: --no-provision leaves + # the machine flagged as not provisioned, so the next vagrant up, whether + # it comes from the next test case or from make ssh, would run it. + no_provision="--no-provision" + e2e_no_provision=1 + # The cluster of a VM which came from a box starts up when the VM boots, + # so it may still be starting, see the wait at the end of this function. + if vm-provisioned-from-box "$vagrantdir"; then + use_cached_box=1 + fi + elif [ ! -f "$vagrantdir/Vagrantfile" ] && vm-box-cache-enabled; then box_file="$BOX_CACHE_DIR/$(vm-box-key "$vmname").box" if [ "$e2e_vm_cache" != "refresh" ] && vm-cached-box-usable "$box_file"; then echo "using cached provisioned VM $box_file..." use_cached_box=1 box_name="$(vm-box-name "$vmname")" distro_img="file://$box_file" - # The box already has everything the playbook installs, and - # kubeadm init cannot run a second time. Keep the provisioner out - # of the Vagrantfile as well: --no-provision leaves the machine - # flagged as not provisioned, so the next vagrant up, whether it - # comes from the next test case or from make ssh, would run it. + # The box already has everything the playbook installs, and kubeadm + # init cannot run a second time. no_provision="--no-provision" e2e_no_provision=1 else package_box=1 fi elif grep -q "^IMAGE_NAME = \"$BOX_NAME_PREFIX/" "$vagrantdir/Vagrantfile" 2>/dev/null; then - # The VM of this output directory was created from a packaged box, so it - # is provisioned whatever this run was asked to do. Skip provisioning - # here too: vagrant would otherwise run it either on a VM which is - # already up, or on one which it is re-importing from that box. + # The VM of this output directory was created from a packaged box, so its + # disk is provisioned however the run which created it ended: a box only + # exists because provisioning succeeded before it was packaged. Skip + # provisioning here too, vagrant would otherwise run it either on a VM + # which is already up, or on one which it is re-importing from that box. echo "the VM of this output directory comes from a packaged box," \ "skipping provisioning..." use_cached_box=1 @@ -547,14 +652,17 @@ vm-setup() { fi fi - # An env file written before this VM was known to come from a box has no - # flag, or a stale one. Keep it in sync, so that a vagrant up which is not - # ours, from make up or make ssh, does not provision the VM either. - if [ -n "$e2e_no_provision" ] && [ -f "$vagrantdir/env" ] && - ! grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env"; then - sed -i 's/^E2E_NO_PROVISION=.*$/E2E_NO_PROVISION=1/' "$vagrantdir/env" - grep -q '^E2E_NO_PROVISION=1$' "$vagrantdir/env" || - echo "E2E_NO_PROVISION=1" >> "$vagrantdir/env" + # An env file written by an earlier run carries the flag of that run, which + # may not be what this one does about provisioning. Keep it in sync, both + # ways: a vagrant up which is not ours, from make up or make ssh, should + # leave a provisioned VM alone, and a run which asks for provisioning needs + # the provisioner in the Vagrantfile. + if [ -f "$vagrantdir/env" ] && + ! grep -q "^E2E_NO_PROVISION=$e2e_no_provision\$" "$vagrantdir/env"; then + sed -i "s/^E2E_NO_PROVISION=.*\$/E2E_NO_PROVISION=$e2e_no_provision/" \ + "$vagrantdir/env" + grep -q "^E2E_NO_PROVISION=" "$vagrantdir/env" || + echo "E2E_NO_PROVISION=$e2e_no_provision" >> "$vagrantdir/env" fi (cd "$vagrantdir"; @@ -611,10 +719,25 @@ EOF mkdir -p "$COMMAND_OUTPUT_DIR" rm -f "$COMMAND_OUTPUT_DIR"/0* + # Record that this VM is provisioned, now that it is up and everything which + # provisions it has run. A VM which came from a box was provisioned before it + # was packaged, so look for the mark the playbook left inside it rather than + # take the box at its word. + # + # This needs vm-command, so it cannot happen before the ssh config above is + # in place. It does come before waiting for the cluster below: a VM which is + # not provisioned has no cluster to wait for, and saying that outright beats + # timing out on a node which is never going to be ready. + if ! vm-provisioned "$vagrantdir"; then + vm-verify-provisioned "$vmname" + vm-mark-provisioned "$vagrantdir" "$vmname" \ + "$(sed -n "s/^IMAGE_NAME = \"\($BOX_NAME_PREFIX\/.*\)\"\$/\1/p" \ + "$vagrantdir/Vagrantfile" 2>/dev/null)" + fi + # A VM which has just booted from a box, or which was brought back up after # being packaged, has a cluster which is still starting up. Wait for it - # before letting the tests run. Both waits need vm-command, so they cannot - # happen before the ssh config above is in place. + # before letting the tests run. if [ -n "$use_cached_box" ] || [ -n "$package_box" ]; then wait-for-node-ready wait-for-dns-ready diff --git a/test/e2e/playbook/provision.yaml b/test/e2e/playbook/provision.yaml index be961c0bf..2b9e930ff 100644 --- a/test/e2e/playbook/provision.yaml +++ b/test/e2e/playbook/provision.yaml @@ -563,3 +563,24 @@ ansible.builtin.systemd: name: kubelet enabled: true + + # Keep this last: it is what says that everything above succeeded. The mark + # travels with the disk image, so a VM created from a "vagrant package"d box + # carries it, and lib/vm.bash can tell a provisioned VM from one whose + # provisioning was interrupted instead of guessing, see PROVISIONED_STAMP. + - name: Record that provisioning completed + ansible.builtin.copy: + dest: /etc/nri-e2e-provisioned + mode: '0644' + content: | + # Written by test/e2e/playbook/provision.yaml when it ran to the end. + # Removing this file makes the e2e framework provision this VM again. + hostname={{ hostname }} + k8s={{ k8s_release }} + k8s_version={{ k8s_version }} + cri={{ cri_runtime }} + containerd={{ containerd_release }} + crio={{ crio_release }} + cni={{ cni_plugin }}{{ cni_release }} + helm={{ helm_release }} + date={{ ansible_date_time.iso8601 }} From baf6d2150de80e68f3c180d9a553868b996136a2 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 28 Aug 2026 22:04:09 +0300 Subject: [PATCH 2/3] e2e: reset the cluster of a VM before provisioning it again. The playbook ends in kubeadm init, which cannot run on a VM which already has a cluster: the ports are taken, the manifests are in place and etcd has data. So provisioning a VM which has been provisioned before failed, whether it was asked for with provision=1 or whether the framework was retrying provisioning that an earlier run had interrupted. Take the existing cluster down with kubeadm reset first. A VM which has not been created, or which never got as far as installing kubeadm, has nothing to reset. This needs the VM to be up: if it cannot be reached the run says so and provisions the VM as it is. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 15 ++++++++++++++- test/e2e/lib/vm.bash | 43 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index a3b803e5f..25eee137e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -161,7 +161,7 @@ Worth knowing: running any test. Raise `CLUSTER_READY_TIMEOUT` (300 seconds by default) if a large topology needs longer. -## Recording that a VM is provisioned +## Provisioning a VM again Provisioning leaves a mark once it has run to the end: `.provisioned` in the output directory, and `/etc/nri-e2e-provisioned` in the VM. Runs read the first @@ -171,6 +171,19 @@ interrupted or which failed therefore leaves the VM marked as what it is, and th next run provisions it again instead of running tests against a VM with no cluster. +Set `provision=1` to provision a VM again on purpose: + +```shell +provision=1 ./run_tests.sh policies.test-suite ~/output-directory +``` + +The playbook ends in `kubeadm init`, which cannot run on a VM which already has a +cluster, so a run which is about to provision an existing VM resets it first with +`kubeadm reset`. This needs the VM to be up; if it is not reachable the framework +says so and provisions the VM as it is, which fails if the earlier provisioning +had got as far as installing the cluster. Bring the VM up (`make up` in the output +directory) before provisioning it again, or start from a clean output directory. + A VM from an output directory or a cached box which predates the marks looks unprovisioned, and the framework says so rather than guess. Start from a clean output directory, or add `e2e_vm_cache=refresh` to replace the box. diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 0cfa2da73..5f1d7fc3e 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -472,6 +472,35 @@ add e2e_vm_cache=refresh to replace the cached box, or provision=1 to provision this VM again." } +vm-kubeadm-reset() { + # Usage: vm-kubeadm-reset VAGRANTDIR + # + # Tear down the Kubernetes cluster of the VM of VAGRANTDIR, if it has one. + # + # Provisioning ends in "kubeadm init", which fails on a VM which already has + # a cluster: the ports are taken, the manifests are in place and etcd has + # data. So provisioning a VM again has to start by resetting whatever is + # there. A VM which is not up, or which never got as far as installing + # kubeadm, has nothing to reset. + local vagrantdir="$1" + local ids=( "$vagrantdir"/.vagrant/machines/*/*/id ) + + if [ ! -f "${ids[0]}" ]; then + # The VM has not been created yet, so there is no cluster in it either. + return 0 + fi + + echo "resetting the Kubernetes cluster of the VM before provisioning it again..." + if ! ( cd "$vagrantdir" && vagrant ssh -c "sudo sh -xc ' + command -v kubeadm > /dev/null || exit 0 + kubeadm reset --force || true + rm -rf /etc/cni/net.d /root/.kube /home/vagrant/.kube + rm -f $VM_PROVISIONED_STAMP'" ); then + echo "WARNING: could not reset the cluster of the VM." \ + "Provisioning it as it is..." >&2 + fi +} + vm-setup() { local output_dir="$1" local vmname="$2" @@ -675,11 +704,19 @@ vm-setup() { error "failed to vagrant init $box_name" fi + # The playbook ends in kubeadm init, which cannot run on a VM which already + # has a cluster. A VM which is about to be provisioned and which has been + # here before may well have one: an earlier run may have been interrupted + # after kubeadm init and before the end of the playbook, or this run may be + # provisioning a working VM again on purpose. Either way, reset it first. + if [ -z "$no_provision" ]; then + vm-kubeadm-reset "$vagrantdir" + fi + # If you want to force provisioning of already provisioned vm, # then you can set provision=1 when calling e2e test script. - # Note that this is not recommended as at least kubeinit - # cannot be called second time. But this could be used - # if the provisioning failed before kubernetes was setup. + # This can be used if the provisioning failed before kubernetes + # was setup, or to reinstall the cluster of a VM from scratch. if [ ! -z "$provision" ]; then if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" vagrant provision ${vagrant_debug:+--debug} || error "failed to provision VM"); then From 7f7b15d77fa94620730781b026fe76be799c01f8 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 28 Aug 2026 22:04:09 +0300 Subject: [PATCH 3/3] e2e: provision a VM which turns out not to be provisioned. A VM without the mark of the playbook cannot run tests, whether its provisioning was interrupted, it comes from an output directory or a box older than the mark, or it was never provisioned at all. Provisioning fixes every one of those, so do that rather than report the VM and give up. Make provision a tri-state so that a run keeps the last word: no leaves an existing VM alone and reports it, 1 provisions it in any case, and the default provisions it only if it turns out to need it. Note that provision=no used to mean the same as provision=1, being non-empty. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 19 +++++-- test/e2e/lib/vm.bash | 127 +++++++++++++++++++++++++++++++++---------- 2 files changed, 113 insertions(+), 33 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 25eee137e..ab3a00a04 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -171,12 +171,25 @@ interrupted or which failed therefore leaves the VM marked as what it is, and th next run provisions it again instead of running tests against a VM with no cluster. -Set `provision=1` to provision a VM again on purpose: +A VM which turns out not to be provisioned is provisioned, whether or not it was +asked for. That covers a VM whose provisioning was interrupted, one from an +output directory or a cached box older than the marks, and one which was never +provisioned at all: none of them can run tests, and provisioning fixes all of +them. `provision` says how much of a say the run has in that: + +| value | meaning | +| --- | --- | +| unset, `auto` | provision an existing VM if it is not provisioned (the default) | +| `1`, `yes` | provision it in any case, even if it already is | +| `no` | leave an existing VM alone, and report it if it is not provisioned | ```shell provision=1 ./run_tests.sh policies.test-suite ~/output-directory ``` +A VM which does not exist yet is provisioned when it is created whatever this +says, since a VM with no cluster is of no use to any test. + The playbook ends in `kubeadm init`, which cannot run on a VM which already has a cluster, so a run which is about to provision an existing VM resets it first with `kubeadm reset`. This needs the VM to be up; if it is not reachable the framework @@ -184,10 +197,6 @@ says so and provisions the VM as it is, which fails if the earlier provisioning had got as far as installing the cluster. Bring the VM up (`make up` in the output directory) before provisioning it again, or start from a clean output directory. -A VM from an output directory or a cached box which predates the marks looks -unprovisioned, and the framework says so rather than guess. Start from a clean -output directory, or add `e2e_vm_cache=refresh` to replace the box. - ## Writing tests A test case is a `code.var.sh` file in a diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 5f1d7fc3e..0d51f0000 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -451,25 +451,106 @@ vm-unmark-provisioned() { rm -f "$1/$PROVISIONED_STAMP" } +vm-provisioning-mode() { + # Usage: vm-provisioning-mode + # + # Print what the provision variable asks of a VM which is already there: + # force: provision it whether it needs it or not (provision=1, provision=yes) + # never: leave it alone even if it is not provisioned (provision=no) + # auto: provision it if it turns out not to be provisioned (the default) + # + # A VM which does not exist yet is provisioned when it is created whatever + # this says: a VM with no cluster in it is of no use to any test. + case "${provision:-}" in + ""|auto) echo auto;; + no|"0"|false|off) echo never;; + *) echo force;; + esac +} + +vm-carries-provisioning-mark() { + # Usage: vm-carries-provisioning-mark + # + # Return success if the VM which is up carries the mark the playbook writes + # when it has run to the end, see VM_PROVISIONED_STAMP. + vm-command-q "[ -f $VM_PROVISIONED_STAMP ]" > /dev/null +} + vm-verify-provisioned() { - # Usage: vm-verify-provisioned VMNAME + # Usage: vm-verify-provisioned VAGRANTDIR VMNAME # - # Fail unless the VM which is up has been provisioned, that is, unless the - # playbook wrote VM_PROVISIONED_STAMP into it. + # Make sure the VM which is up is provisioned, and provision it if it is not. # - # This is what catches a VM whose provisioning never finished, and a cached - # box which was packaged from one. - local vmname="$1" + # A VM without the mark of the playbook is a VM whose provisioning never + # finished, one which comes from a box or an output directory older than the + # mark, or one which was never provisioned at all. None of those can run + # tests, and all of them are fixed by provisioning the VM, so do that rather + # than hand the run a VM which does not work. provision=no says to keep hands + # off, in which case there is nothing to do but report it. + local vagrantdir="$1" vmname="$2" - if vm-command-q "[ -f $VM_PROVISIONED_STAMP ]" > /dev/null; then + if vm-carries-provisioning-mark; then return 0 fi - error "VM $vmname does not look provisioned: $VM_PROVISIONED_STAMP is missing. + if [ "$(vm-provisioning-mode)" == "never" ]; then + error "VM $vmname is not provisioned: $VM_PROVISIONED_STAMP is missing. Its provisioning either never finished, or the VM comes from a cached box or an -output directory which predates this check. Start from a clean output directory, -add e2e_vm_cache=refresh to replace the cached box, or provision=1 to provision -this VM again." +output directory which predates this mark. provision=no asks to leave the VM +alone, so this run cannot fix it. Drop provision=no to have it provisioned, start +from a clean output directory, or add e2e_vm_cache=refresh to replace the box." + fi + + echo "VM $vmname is not provisioned, $VM_PROVISIONED_STAMP is missing." \ + "Provisioning it..." + vm-provision-now "$vagrantdir" "$vmname" + + vm-carries-provisioning-mark || + error "provisioning VM $vmname left no $VM_PROVISIONED_STAMP behind" +} + +vm-provision-now() { + # Usage: vm-provision-now VAGRANTDIR VMNAME + # + # Run the provisioning playbook on the VM of VAGRANTDIR, which is up. + local vagrantdir="$1" vmname="$2" + + # The provisioner may have been kept out of the Vagrantfile of this VM, in + # which case vagrant has nothing to run. Put it back first. + vm-set-no-provision-flag "$vagrantdir" "" + + # The playbook ends in kubeadm init, so whatever cluster the VM has now has + # to go. + vm-kubeadm-reset "$vagrantdir" + + ( cd "$vagrantdir" && + export ANSIBLE_PIPELINING=True && + export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" && + vagrant provision ${vagrant_debug:+--debug} ) || + error "failed to provision VM $vmname" +} + +vm-set-no-provision-flag() { + # Usage: vm-set-no-provision-flag VAGRANTDIR VALUE + # + # Set E2E_NO_PROVISION in the env file of VAGRANTDIR, which is what decides + # whether the Vagrantfile has the provisioner in it at all. + # + # An env file written by an earlier run carries the flag of that run, which + # may not be what this one does about provisioning. Keep it in sync, both + # ways: a vagrant up which is not ours, from make up or make ssh, should + # leave a provisioned VM alone, and provisioning needs the provisioner in + # the Vagrantfile. + local vagrantdir="$1" value="$2" + + if [ ! -f "$vagrantdir/env" ] || + grep -q "^E2E_NO_PROVISION=$value\$" "$vagrantdir/env"; then + return 0 + fi + + sed -i "s/^E2E_NO_PROVISION=.*\$/E2E_NO_PROVISION=$value/" "$vagrantdir/env" + grep -q "^E2E_NO_PROVISION=" "$vagrantdir/env" || + echo "E2E_NO_PROVISION=$value" >> "$vagrantdir/env" } vm-kubeadm-reset() { @@ -515,6 +596,7 @@ vm-setup() { local efi_code efi_vars kind local box_name="$distro" box_file="" package_box="" use_cached_box="" local no_provision="" e2e_no_provision="" + local provisioning_mode="$(vm-provisioning-mode)" # Decide what this run does about provisioning. # @@ -530,7 +612,7 @@ vm-setup() { # directory which already has a Vagrantfile keeps the VM and the box it was # created from, so start from a clean output directory to benefit from the # cache. - if [ -n "$provision" ]; then + if [ "$provisioning_mode" == "force" ]; then # Provisioning was asked for explicitly, so provision whatever is here. # Until that succeeds this VM does not count as provisioned. vm-unmark-provisioned "$vagrantdir" @@ -681,18 +763,7 @@ vm-setup() { fi fi - # An env file written by an earlier run carries the flag of that run, which - # may not be what this one does about provisioning. Keep it in sync, both - # ways: a vagrant up which is not ours, from make up or make ssh, should - # leave a provisioned VM alone, and a run which asks for provisioning needs - # the provisioner in the Vagrantfile. - if [ -f "$vagrantdir/env" ] && - ! grep -q "^E2E_NO_PROVISION=$e2e_no_provision\$" "$vagrantdir/env"; then - sed -i "s/^E2E_NO_PROVISION=.*\$/E2E_NO_PROVISION=$e2e_no_provision/" \ - "$vagrantdir/env" - grep -q "^E2E_NO_PROVISION=" "$vagrantdir/env" || - echo "E2E_NO_PROVISION=$e2e_no_provision" >> "$vagrantdir/env" - fi + vm-set-no-provision-flag "$vagrantdir" "$e2e_no_provision" (cd "$vagrantdir"; export ANSIBLE_PIPELINING=True; @@ -713,11 +784,11 @@ vm-setup() { vm-kubeadm-reset "$vagrantdir" fi - # If you want to force provisioning of already provisioned vm, + # If you want to force provisioning of an already provisioned vm, # then you can set provision=1 when calling e2e test script. # This can be used if the provisioning failed before kubernetes # was setup, or to reinstall the cluster of a VM from scratch. - if [ ! -z "$provision" ]; then + if [ "$provisioning_mode" == "force" ]; then if ! (export ANSIBLE_SSH_ARGS="$SSH_PERSIST_OPTS" vagrant provision ${vagrant_debug:+--debug} || error "failed to provision VM"); then exit 1 @@ -763,10 +834,10 @@ EOF # # This needs vm-command, so it cannot happen before the ssh config above is # in place. It does come before waiting for the cluster below: a VM which is - # not provisioned has no cluster to wait for, and saying that outright beats + # not provisioned has no cluster to wait for, and provisioning it now beats # timing out on a node which is never going to be ready. if ! vm-provisioned "$vagrantdir"; then - vm-verify-provisioned "$vmname" + vm-verify-provisioned "$vagrantdir" "$vmname" vm-mark-provisioned "$vagrantdir" "$vmname" \ "$(sed -n "s/^IMAGE_NAME = \"\($BOX_NAME_PREFIX\/.*\)\"\$/\1/p" \ "$vagrantdir/Vagrantfile" 2>/dev/null)"