From 5fcf63f0971018a219f6745a05cd6213c6fe5628 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 27 Aug 2026 10:18:15 +0300 Subject: [PATCH 01/31] e2e: add lib/test.bash test helper library. Test cases re-implement the same helper functions over and over again, often in subtly different variants. Add a library for lifting those out into shared helpers, together with the polling helper which most of them turn out to need. The library is not sourced by run.sh. Instead, run_tests.sh seeds the *.source.sh chain with it, which means that - the helpers are sourced into the same subshell that evaluates the test case code and nowhere else, so they cannot clash with the script API of run.sh, and - any *.source.sh file can override a helper defined in the library, as those are sourced after it. run.sh still reads the library, but only to scrape the documentation of the helpers for "run.sh help". retry-until is the host side counterpart of vm-run-until: it evaluates a snippet until it succeeds instead of running a command in the VM. Nine test cases hand-roll that loop with different variable names, different off-by-one behaviour, and in one case with a retry counter which is never incremented. Unlike those loops it never fails the test by itself, so that callers can choose between error, command-error and tolerating the timeout. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 20 ++++++++++++ test/e2e/lib/test.bash | 70 ++++++++++++++++++++++++++++++++++++++++++ test/e2e/run.sh | 4 ++- test/e2e/run_tests.sh | 9 +++++- 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 test/e2e/lib/test.bash diff --git a/test/e2e/README.md b/test/e2e/README.md index 444e34c08..f71d1c6e2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -96,3 +96,23 @@ Before running E2E tests ensure that you have all the required components locall policies.test-suite balloons test09-isolated : PASS policies.test-suite balloons test10-health-checking : PASS ``` + +## Writing tests + +A test case is a `code.var.sh` file in a +`TEST-SUITE/POLICY/TOPOLOGY/TEST/` directory. In addition to the script API of +`run.sh` (run `./run.sh help` to list it), test cases have shared helpers +available: + +- `lib/test.bash` contains helpers that are useful for more than one test case, + for instance for waiting for a condition, cleaning up pods and namespaces, + inspecting container states, or reading the log of the plugin. Add a helper + here if a second test case needs it. `./run.sh help` documents these, too. + +- `TEST-SUITE/POLICY/*.source.sh` files contain policy-specific helpers, for + instance `policies.test-suite/balloons/nrt.source.sh`. + +`*.source.sh` files are sourced before the test case code, starting from the +test suite directory and ending in the test case directory, `lib/test.bash` +being the first one. Therefore a test case, or all test cases of a policy or a +topology, can override any shared helper simply by redefining it. diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash new file mode 100644 index 000000000..523efaef5 --- /dev/null +++ b/test/e2e/lib/test.bash @@ -0,0 +1,70 @@ +# Shared helpers for test cases (code.var.sh files). +# +# This library is not sourced by run.sh. Instead, run_tests.sh seeds the +# *.source.sh chain with it, so that +# +# - the helpers live in the same subshell as the test code and nowhere else, +# and cannot clash with the script API of run.sh, and +# - any *.source.sh file (test suite, policy, topology or test case level) can +# override a helper defined here, because those are sourced after this file. +# +# Helpers here are documented in the same format as the script API of run.sh, +# that is, a "# script API" marked function followed by indented comment lines. +# Run "./run.sh help" to print the documentation of all available functions. +# +# Add a helper here if more than one test case needs it. Keep policy-specific +# helpers in POLICY/*.source.sh instead. + +# Fail if this was sourced without the primitives it builds on. +if ! type -t vm-command >/dev/null; then + echo "error: lib/test.bash: lib/vm.bash must be sourced first" >&2 + return 1 +fi + +### +### Waiting for things to happen +### + +retry-until() { # script API + # Usage: retry-until [--timeout SECS] [--interval SECS] [--message MSG] SNIPPET + # + # Evaluate SNIPPET on the host repeatedly until it exits with status 0. + # Give up after SECS seconds (default 30), waiting SECS seconds between + # the attempts (default 1). Both must be integers. Print MSG before the + # first attempt and when giving up. + # + # Return 0 if SNIPPET succeeded, non-zero if it did not. Never fails the + # test, so that the caller can choose between error, command-error and + # ignoring the timeout. + # + # SNIPPET is evaluated, so single-quote it to have the values it refers to + # re-read on every attempt: + # retry-until --timeout 10 'vm-command "kubectl get pod $pod"' + # + # This is the host side counterpart of vm-run-until. + local timeout=30 interval=1 message="" elapsed=0 + while [ "${1#--}" != "$1" ]; do + case "$1" in + --timeout) timeout="$2"; shift 2;; + --interval) interval="$2"; shift 2;; + --message) message="$2"; shift 2;; + --) shift; break;; + *) error "retry-until: unknown option \"$1\"";; + esac + done + if [ -n "$message" ]; then + echo "waiting for: $message" + fi + while true; do + if eval "$*"; then + return 0 + fi + if [ "$elapsed" -ge "$timeout" ]; then + break + fi + sleep "$interval" + elapsed=$(( elapsed + interval )) + done + echo "timeout after ${timeout}s${message:+ waiting for: $message}" >&2 + return 1 +} diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 4e015e593..ff6fe9df8 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -249,7 +249,9 @@ if [ "$1" == "runtime-logs" ]; then exit fi -script_source="$(< "$0") $(< "$LIB_DIR/vm.bash")" +# Note: lib/test.bash is read for its documentation only. It is not sourced +# here, but by the test case itself, see run_tests.sh. +script_source="$(< "$0") $(< "$LIB_DIR/vm.bash") $(< "$LIB_DIR/test.bash")" help() { # script API # Usage: help [FUNCTION|all] diff --git a/test/e2e/run_tests.sh b/test/e2e/run_tests.sh index 4a36ecb88..c36764f40 100755 --- a/test/e2e/run_tests.sh +++ b/test/e2e/run_tests.sh @@ -169,7 +169,9 @@ echo " TESTS_TOPOLOGY_FILTER=$TESTS_TOPOLOGY_FILTER" echo " TESTS_TEST_FILTER=$TESTS_TEST_FILTER" echo " skip long tests: $SKIP_LONG_TESTS" -source "$TESTS_ROOT_DIR"/../lib/vm.bash +E2E_LIB_DIR=$(realpath "$TESTS_ROOT_DIR/../lib") + +source "$E2E_LIB_DIR"/vm.bash cleanup() { rm -rf "$summary_dir" @@ -180,6 +182,11 @@ trap cleanup TERM EXIT QUIT summary_file="$summary_dir/summary.txt" echo -n "" > "$summary_file" +# Shared test helpers are the root of the *.source.sh chain: they are sourced +# before any test suite, policy, topology or test case level *.source.sh file, +# so that any of those can override a helper. +source_libs="source \"$E2E_LIB_DIR/test.bash\"" + export-and-source-dir "$TESTS_ROOT_DIR" TEST_SUITE_NAME="$(basename $TESTS_ROOT_DIR)" From 90e3dc71f527efd20cd7c0aa850854a6ad03f0a1 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 27 Aug 2026 10:18:29 +0300 Subject: [PATCH 02/31] e2e: add pod and namespace cleanup helpers, use them in the tests. Thirty test cases define a cleanup function, and they are all built from the same handful of primitives: delete all pods, delete named pods in a namespace, create and delete a list of namespaces, and remove the cache file of the policy. The variants differ mostly in whether they tolerate errors, which is accidental rather than intentional. The helpers pass --ignore-not-found and never fail, so tests no longer need to sprinkle "|| true", "|| :" and "return 0" around them, and calling cleanup at the beginning of a test no longer logs errors about pods and namespaces which are not there yet. kill-test-processes brackets the space in its pkill patterns. Without that, "pkill -9 -f 'echo pod'" also matches the command line of the shell which runs it, and kills it. That is what the cleanup of test31-duplicate-disambiguation used to do. Only cleanup and teardown call sites are converted. Pod deletions in the middle of a test are part of the scenario, and deliberately keep failing if the pod is not there. The awk based pod selection of the fuzz tests is not expressible with the helpers and is left as it is. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 66 +++++++++++++++++++ .../n4c16/test01-basic-placement/code.var.sh | 7 +- .../test02-prometheus-metrics/code.var.sh | 3 +- .../n4c16/test03-reserved/code.var.sh | 18 ++--- .../balloons/n4c16/test04-groupby/code.var.sh | 8 +-- .../n4c16/test05-namespace/code.var.sh | 18 +---- .../n4c16/test06-update-config/code.var.sh | 14 ++-- .../n4c16/test07-maxballoons/code.var.sh | 3 +- .../n4c16/test10-allocator-opts/code.var.sh | 3 +- .../n4c16/test11-match-expression/code.var.sh | 3 +- .../n4c16/test13-cacheclusters/code.var.sh | 4 +- .../n4c16/test15-loadclasses/code.var.sh | 4 +- .../test16-composite-balloons/code.var.sh | 3 +- .../test17-cstates-scheduling/code.var.sh | 2 +- .../n4c16/test18-turbo-priority/code.var.sh | 2 +- .../balloons/n4c16/test19-pct/code.var.sh | 2 +- .../n4c16/test22-isolcpus/code.var.sh | 6 +- .../n4c16/test23-available-cpus/code.var.sh | 8 +-- .../n4c16/test24-podresources/code.var.sh | 3 +- .../balloons/n4c16/test25-irq/code.var.sh | 2 +- .../n4c16/test00-basic-placement/code.var.sh | 9 ++- .../test05-reserved-resources/code.var.sh | 2 +- .../test07-mixed-allocations/code.var.sh | 4 +- .../test08-cpuprio-allocation/code.var.sh | 2 +- .../n4c16/test09-container-exit/code.var.sh | 2 +- .../code.var.sh | 10 +-- .../code.var.sh | 5 +- .../n4c16/test14-burstable/code.var.sh | 2 +- .../code.var.sh | 2 +- .../test15-busy-shared-pools/code.var.sh | 4 +- .../test16-idle-shared-pools/code.var.sh | 4 +- .../test17-scheduling-classes/code.var.sh | 8 +-- .../n4c16/test18-strict-alignment/code.var.sh | 4 +- .../n4c16/test19-cpuclass/code.var.sh | 2 +- .../n4c16/test25-irq/code.var.sh | 12 ++-- .../n4c16/test30-numa-disabled/code.var.sh | 2 +- .../code.var.sh | 9 ++- .../n4c16/test40-otel-logging/code.var.sh | 2 +- 38 files changed, 146 insertions(+), 118 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 523efaef5..ad5de195e 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -68,3 +68,69 @@ retry-until() { # script API echo "timeout after ${timeout}s${message:+ waiting for: $message}" >&2 return 1 } + +### +### Cleaning up +### + +delete-pods() { # script API + # Usage: delete-pods [-n NAMESPACE] {--all | POD...} + # + # Delete pods immediately, ignoring pods which do not exist. Delete the + # pods in NAMESPACE, or in the default namespace if -n is not given. + # + # Never fails, so this is safe to call both before and after a test. + local ns="" + if [ "$1" == "-n" ]; then + ns="-n $2" + shift 2 + fi + if [ $# -eq 0 ]; then + error "delete-pods: expected --all or a list of pods" + fi + vm-command "kubectl delete pods $ns $* --now --ignore-not-found=true" || : +} + +create-namespaces() { # script API + # Usage: create-namespaces NAMESPACE... + # + # Create namespaces unless they already exist. Never fails. + local ns + for ns in "$@"; do + vm-command "kubectl get namespace $ns > /dev/null 2>&1 || kubectl create namespace $ns" || : + done +} + +delete-namespaces() { # script API + # Usage: delete-namespaces NAMESPACE... + # + # Delete all pods in the namespaces, then the namespaces themselves. + # Namespaces which do not exist are ignored. Never fails. + local ns + for ns in "$@"; do + vm-command "kubectl delete pods -n $ns --all --now --ignore-not-found=true + kubectl delete namespace $ns --now --ignore-not-found=true" || : + done +} + +remove-policy-cache() { # script API + # Usage: remove-policy-cache + # + # Remove the cache file of the resource policy from the node. Never fails. + # + # Use this to prevent the cache of a previously running policy from + # affecting the policy which the test launches. + vm-command "rm -f /var/lib/nri-resource-policy/cache" || : +} + +kill-test-processes() { # script API + # Usage: kill-test-processes + # + # Kill leftover processes of test containers in the VM. Never fails. + # + # The patterns match the commands which the pod templates in files/ run. + # They use a bracket expression so that they do not match the command + # line of the shell which runs pkill. + vm-command 'pkill -9 -f "sleep[ ]inf" + pkill -9 -f "echo[ ]pod"' || : +} diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh index 297da9440..7c05f730e 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh @@ -5,8 +5,9 @@ helm-terminate helm_config=${TEST_DIR}/../../helm-config.yaml helm-launch balloons cleanup() { - vm-command "kubectl delete pods -n kube-system pod0; kubectl delete pods -n three --all --now; kubectl delete pods --all --now; kubectl delete namespace three" - return 0 + delete-pods -n kube-system pod0 + delete-pods --all + delete-namespaces three } cleanup @@ -36,7 +37,7 @@ verify 'len(cpus["pod2c0"]) == 2' \ # pod3: fits exactly on a single three-cpu instance. No need to create # new balloon even if spreading pods is preferred. CPUREQ="1500m" MEMREQ="100M" CPULIM="1500m" MEMLIM="100M" -vm-command "kubectl create namespace three" +create-namespaces three namespace="three" CONTCOUNT=2 create balloons-busybox report allowed verify 'cpus["pod3c0"] == cpus["pod3c1"]' \ diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh index df5aef629..199884755 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh @@ -1,8 +1,7 @@ # This test verifies prometheus metrics from the balloons policy. cleanup() { - vm-command "kubectl delete pods --all --now" - return 0 + delete-pods --all } cleanup diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh index d26c169f7..bd2fff22a 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh @@ -2,24 +2,14 @@ helm-terminate helm_config=${TEST_DIR}/balloons-reserved.cfg helm-launch balloons cleanup() { - vm-command \ - "kubectl delete pod -n kube-system --now pod0 - kubectl delete pod -n monitor-mypods --now pod1 - kubectl delete pod -n system-logs --now pod2 - kubectl delete pod -n kube-system --now pod3 - kubectl delete pods --now pod4 pod5 pod6 - kubectl delete pod -n kube-system --now pod7 - kubectl delete namespace monitor-mypods - kubectl delete namespace system-logs - kubectl delete namespace my-exact-name" - return 0 + delete-pods -n kube-system pod0 pod3 pod7 + delete-pods pod4 pod5 pod6 + delete-namespaces monitor-mypods system-logs my-exact-name } cleanup -vm-command "kubectl create namespace monitor-mypods" -vm-command "kubectl create namespace system-logs" -vm-command "kubectl create namespace my-exact-name" +create-namespaces monitor-mypods system-logs my-exact-name # pod0: kube-system CPUREQ="100m" MEMREQ="100M" CPULIM="100m" MEMLIM="100M" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh index de8cc1837..5ceffaa53 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh @@ -7,10 +7,8 @@ helm_config=$TEST_DIR/balloons-groupby.cfg helm-launch balloons testns=e2e-balloons-test04 cleanup() { - vm-command "kubectl delete pods --all --now; \ - kubectl delete pods -n $testns --all --now; \ - kubectl delete namespace $testns; \ - true" + delete-pods --all + delete-namespaces "$testns" } cleanup @@ -30,7 +28,7 @@ create balloons-busybox # pod3c0 POD_LABEL='balloon-instance: g1' -vm-command "kubectl create namespace $testns" +create-namespaces "$testns" namespace=$testns create balloons-busybox report allowed diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh index 12305eff2..734d2d707 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh @@ -2,24 +2,12 @@ helm-terminate helm_config=${TEST_DIR}/balloons-namespace.cfg helm-launch balloons cleanup() { - vm-command \ - "kubectl delete pods -n e2e-a --all --now - kubectl delete pods -n e2e-b --all --now - kubectl delete pods -n e2e-c --all --now - kubectl delete pods -n e2e-d --all --now - kubectl delete pods --all --now - kubectl delete namespace e2e-a - kubectl delete namespace e2e-b - kubectl delete namespace e2e-c - kubectl delete namespace e2e-d" - return 0 + delete-pods --all + delete-namespaces e2e-a e2e-b e2e-c e2e-d } cleanup -vm-command "kubectl create namespace e2e-a" -vm-command "kubectl create namespace e2e-b" -vm-command "kubectl create namespace e2e-c" -vm-command "kubectl create namespace e2e-d" +create-namespaces e2e-a e2e-b e2e-c e2e-d # pod0: create in the default namespace, both containers go to nsballoon[0] CPUREQ="" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh index 6ff15e8bf..8284a398b 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh @@ -7,17 +7,14 @@ helm_config=$TEST_DIR/initial-balloons-config.cfg helm-launch balloons testns=e2e-balloons-test06 cleanup() { - vm-command "kubectl delete pods --all --now; \ - kubectl delete pods -n $testns --all --now; \ - kubectl delete pods -n btype1ns0 --all --now; \ - kubectl delete namespace $testns || :; \ - kubectl delete namespace btype1ns0 || :; \ - kubectl -n kube-system delete configmap nri-resource-policy-config.default || :" + delete-pods --all + delete-namespaces "$testns" btype1ns0 + vm-command "kubectl -n kube-system delete configmap nri-resource-policy-config.default --ignore-not-found=true" || : helm-terminate # Just in case the cache says that the policy is "topology-aware" # (from earlier tests) then remove the cache to force "balloons" policy - vm-command "rm -f /var/lib/nri-resource-policy/cache" || true + remove-policy-cache } apply-configmap() { @@ -29,8 +26,7 @@ apply-configmap() { cleanup helm_config=$TEST_DIR/initial-balloons-config.cfg helm-launch balloons -vm-command "kubectl create namespace $testns" -vm-command "kubectl create namespace btype1ns0" +create-namespaces "$testns" btype1ns0 AVAILABLE_CPU="cpuset:0,4-15" BTYPE2_NAMESPACE0='"*"' BTYPE1_MAXCPUS='0' apply-configmap sleep 3 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh index 149a29763..3f4350117 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh @@ -1,6 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" - return 0 + delete-pods --all } cleanup diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh index c82b4072c..e93d084f0 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh @@ -1,6 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now --wait" - return 0 + delete-pods --all } cleanup diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh index 551a6b7eb..8ddb15398 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh @@ -5,8 +5,7 @@ helm-terminate helm_config=${TEST_DIR}/../../match-config.yaml helm-launch balloons cleanup() { - vm-command "kubectl delete pods --all --now" - return 0 + delete-pods --all } cleanup diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh index 44a838ac7..6af65f891 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh @@ -2,9 +2,9 @@ helm-terminate helm_config=$TEST_DIR/balloons-4cpu-cacheclusters.cfg helm-launch balloons cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate - vm-command "rm -f /var/lib/nri-resource-policy/cache" || true + remove-policy-cache } # pod0c{0,1,2}: one container per free L2 group diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh index 5e801f0ee..c33937ef4 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh @@ -6,9 +6,9 @@ helm-terminate helm_config=$TEST_DIR/balloons-loadclasses.cfg helm-launch balloons cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate - vm-command "rm -f /var/lib/nri-resource-policy/cache" || true + remove-policy-cache } # Policy's allocatorTopologyBalancing is false, so CPU allocations are diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh index ca0e6c41c..7e080dc12 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh @@ -4,7 +4,8 @@ helm-terminate helm_config=$TEST_DIR/balloons-composite.cfg helm-launch balloons cleanup() { - vm-command "kubectl delete -n kube-system pod pod2 --now; kubectl delete pods --all --now" + delete-pods -n kube-system pod2 + delete-pods --all } verify-nrt() { diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index dda1b1661..47b541424 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -66,7 +66,7 @@ verify-cstates-no-writes() { } cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all } echo "verify that all c-states of all available CPUs are enabled" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh index 5382811ab..6e0e26a03 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh @@ -578,5 +578,5 @@ if ! grep "on cpu $reserved_cpu\$" <<< "$back_lines" | grep 'default-turbo' | gr fi echo "turboDomain back to package: cpu $reserved_cpu (default-turbo) at max=3800000 as expected" -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index f7c7f4833..2b667f2cf 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -358,7 +358,7 @@ wait-assert-log-contains 'associated cpus .* to CLOS 1' "CPUs not associated to assert-log-not-contains 'PrepareManagedMode done' "PrepareManagedMode unexpectedly called in assoc-only mode" assert-log-not-contains 'EnableCP done' "EnableCP unexpectedly called in assoc-only mode" -vm-command "kubectl delete pods --all --now" || true +delete-pods --all helm-terminate ############################################################################### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh index 2599a1a2c..0c149e183 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh @@ -10,8 +10,8 @@ restart-kubelet() { } cleanup-pods() { - vm-command "kubectl delete pod --all --now" - vm-command "kubectl delete namespace $ns --now" + delete-pods --all + delete-namespaces "$ns" } cleanup() { @@ -40,7 +40,7 @@ vm-command "grep isolcpus=0,1 /proc/cmdline" || { helm-terminate helm_config=${TEST_DIR}/balloons-isolcpus.cfg helm-launch balloons -vm-command "kubectl create namespace $ns" +create-namespaces "$ns" # pod0: should run on non-isolated CPUs CONTCOUNT=2 namespace="default" create balloons-busybox diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh index 264df999e..e01d7ffe9 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh @@ -1,12 +1,10 @@ cleanup() { - vm-command \ - "kubectl -n kube-system delete pod pod0 --now && \ - kubectl -n reserved delete pod pod1 --now || true && \ - kubectl delete ns reserved --now" + delete-pods -n kube-system pod0 + delete-namespaces reserved } cleanup -vm-command "kubectl create namespace reserved || true" +create-namespaces reserved helm-terminate helm_config=${TEST_DIR}/balloons-excluded-cpusets.cfg helm-launch balloons diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh index a6064fc7e..d4d783ddf 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh @@ -6,7 +6,7 @@ helm_config=$TEST_DIR/balloons-podresources.cfg helm-launch balloons cleanup() { vm-command 'pidof fake-device-plugin && kill $(pidof fake-device-plugin) && sleep 1' - vm-command "kubectl delete pods --all --now" || true + delete-pods --all } # verify-podres-locality RESOURCE CONTAINER... @@ -171,5 +171,4 @@ report allowed verify-podres-locality "telco.com/nic" pod3c0 cleanup -vm-command "kubectl delete pods --all --now" helm-terminate diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index c8d2e2058..67180769f 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -71,7 +71,7 @@ ACPI_IRQ=$COMMAND_OUTPUT ALL_CPUS="$(expand-cpulist 0-15)" cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all vm-command "for f in /proc/irq/*/smp_affinity_list ; do echo 0-15 | tee $f >&/dev/null; done" } diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh index 9d68c3481..98703f758 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh @@ -1,8 +1,8 @@ # Make sure all the pods in default namespace are cleared so we get a fresh start -vm-command "kubectl delete pods --all --now" +delete-pods --all # Remove also any leftover test pods from kube-system -vm-command "kubectl delete pods pod0 pod1 pod2 pod3 pod4 pod5 --ignore-not-found=true --now -n kube-system" +delete-pods -n kube-system pod0 pod1 pod2 pod3 pod4 pod5 # Cleanup kernel commandline, otherwise isolcpus will affect CPU # pinning and cause false negatives from other tests on this VM. @@ -97,7 +97,7 @@ vm-command "kubectl delete pods --all --now" helm-terminate helm_config=$(COLOCATE_NAMESPACES=true instantiate helm-config.yaml) helm-launch topology-aware -vm-command "kubectl create namespace test-ns" +create-namespaces test-ns CONTCOUNT=1 CPU=100m namespace=test-ns create guaranteed CONTCOUNT=1 CPU=100m namespace=test-ns create guaranteed @@ -108,8 +108,7 @@ verify \ 'cpus["pod7c0"] == cpus["pod5c0"]' \ 'cpus["pod7c1"] == cpus["pod5c0"]' -vm-command "kubectl delete pods -n test-ns --all --now" -vm-command "kubectl delete namespace test-ns" +delete-namespaces test-ns # Restore default test configuration, restart nri-resource-policy. helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh index ab5e53bfc..134b3fdf1 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh @@ -11,7 +11,7 @@ AVAILABLE_CPU="cpuset:4-7,8-13" # if exiting with success. Otherwise leave the pod running for # debugging in case of a failure. cleanup-kube-system() { - ( vm-command "kubectl delete pods pod0 pod1 pod2 pod3 pod4 pod5 -n kube-system --now --ignore-not-found=true" ) || true + delete-pods -n kube-system pod0 pod1 pod2 pod3 pod4 pod5 } cleanup-kube-system diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh index 951a465e0..9e292fcf3 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh @@ -3,10 +3,10 @@ helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware cleanup-test-pods() { # Make sure all the pods in default namespace are cleared so we get a fresh start - vm-command "kubectl delete pods --all --now" + delete-pods --all # Remove also any leftover test pods from kube-system - vm-command "kubectl delete pods pod0 pod1 pod2 pod3 pod4 pod5 --ignore-not-found=true --now -n kube-system" + delete-pods -n kube-system pod0 pod1 pod2 pod3 pod4 pod5 } cleanup-test-pods diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test08-cpuprio-allocation/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test08-cpuprio-allocation/code.var.sh index ab600235a..228d35e7d 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test08-cpuprio-allocation/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test08-cpuprio-allocation/code.var.sh @@ -52,4 +52,4 @@ verify \ 'cpus["pod1c1"] not in [ {"cpu01"}, {"cpu04"}, {"cpu05"} ]' \ 'cpus["pod1c2"] in [ {"cpu01"}, {"cpu04"}, {"cpu05"} ]' \ -vm-command "kubectl delete pods --all --now" +delete-pods --all diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test09-container-exit/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test09-container-exit/code.var.sh index 286d52719..5885a4859 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test09-container-exit/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test09-container-exit/code.var.sh @@ -5,7 +5,7 @@ helm-terminate helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware # Make sure all the pods in default namespace are cleared so we get a fresh start -vm-command "kubectl delete pods --all --now" +delete-pods --all CONTCOUNT=1 CPU=1000m MEM=64M create guaranteed report allowed diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/code.var.sh index bc487bbf7..4d6377367 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/code.var.sh @@ -1,15 +1,15 @@ # Test that # - containers marked in ReservedPoolNamespaces option pinned on Reserved CPUs. -( vm-command "kubectl create namespace reserved-test" ) || true +create-namespaces reserved-test # This script will create pods to the reserved and default namespace. # Make sure the namespace is clear when starting the test and clean it up # if exiting with success. Otherwise leave the pod running for # debugging in case of a failure. cleanup-test-pods() { - ( vm-command "kubectl delete pods pod0 -n kube-system --now" ) || true - ( vm-command "kubectl delete pods pod1 --now" ) || true + delete-pods -n kube-system pod0 + delete-pods pod1 } cleanup-test-pods @@ -32,10 +32,10 @@ cleanup-test-pods # - containers that are annotated to opt-put that are pinned elsewhere, and # - containers that are namespace-assigned and annotated to reserved pools are pinned there -( vm-command "kubectl create namespace foobar" ) || true +create-namespaces foobar cleanup-foobar-namespace() { - ( vm-command "kubectl delete pods -n foobar --all" ) || true + delete-pods -n foobar --all } cleanup-foobar-namespace diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/code.var.sh index 315b4f884..f5bc61ef4 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/code.var.sh @@ -4,10 +4,7 @@ # - memory.preserve cleanup-test-pods() { - ( vm-command "kubectl delete pods pod0 --now" ) || true - ( vm-command "kubectl delete pods pod1 --now" ) || true - ( vm-command "kubectl delete pods pod2 --now" ) || true - ( vm-command "kubectl delete pods pod3 --now" ) || true + delete-pods pod0 pod1 pod2 pod3 } cleanup-test-pods diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test14-burstable/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test14-burstable/code.var.sh index b274b0248..c968c22b0 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test14-burstable/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test14-burstable/code.var.sh @@ -85,6 +85,6 @@ verify \ 'len(nodes["pod8c0"]) == 1' \ 'len(nodes["pod8c1"]) == 2' -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test14-global-shared-preference/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test14-global-shared-preference/code.var.sh index 84d7b6880..331c64e0b 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test14-global-shared-preference/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test14-global-shared-preference/code.var.sh @@ -1,6 +1,6 @@ cleanup-test-pods() { # Make sure all the pods in default namespace are cleared so we get a fresh start - vm-command "kubectl delete pods --all --now" + delete-pods --all } # restart with a global shared CPU allocation preference diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test15-busy-shared-pools/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test15-busy-shared-pools/code.var.sh index 709e0a505..3c73dcb12 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test15-busy-shared-pools/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test15-busy-shared-pools/code.var.sh @@ -1,4 +1,4 @@ -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate helm_config=$(COLOCATE_PODS=false instantiate helm-config.yaml) helm-launch topology-aware @@ -30,5 +30,5 @@ CONTCOUNT=4 CPUREQ=250m CPULIM=750m create burstable CPU=4 MEM=100M CONTCOUNT=1 create guaranteed verify 'disjoint_sets(cpus["pod4c0"],cpus["pod4c1"],cpus["pod4c2"],cpus["pod4c3"],cpus["pod5c0"])' -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test16-idle-shared-pools/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test16-idle-shared-pools/code.var.sh index d08aa36da..cc7164f48 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test16-idle-shared-pools/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test16-idle-shared-pools/code.var.sh @@ -1,4 +1,4 @@ -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate helm_config=$(COLOCATE_PODS=false instantiate helm-config.yaml) helm-launch topology-aware @@ -43,5 +43,5 @@ verify 'len(cpus["pod4c0"]) == 4' \ 'len(nodes["pod4c0"]) == 1' verify 'disjoint_sets(nodes["pod3c0"],nodes["pod3c1"],nodes["pod4c0"])' -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh index 2f13d46b9..cd1cba6b3 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh @@ -1,6 +1,6 @@ cleanup() { - vm-command "kubectl delete pods --all --now" - vm-command "kubectl delete namespaces highprio lowprio --now --ignore-not-found" + delete-pods --all + delete-namespaces highprio lowprio } verify-sched() { @@ -109,7 +109,7 @@ vm-command "kubectl delete pods --all --now" # # First in a namespace with default highprio scheduling class. -vm-command "kubectl create namespace highprio" +create-namespaces highprio ANN0="scheduling-class.resource-policy.nri.io/container.pod4c0: lowprio" \ CONTCOUNT=2 namespace=highprio create burstable @@ -120,7 +120,7 @@ verify 'len(cpus["pod4c1"]) != 1' expected_policy=$SCHED_FIFO expected_prio=$((99 - 10)) verify-sched pod4c1 # Then in a namespace with default lowprio scheduling class. -vm-command "kubectl create namespace lowprio" +create-namespaces lowprio ANN0="scheduling-class.resource-policy.nri.io/container.pod5c0: highprio" \ CONTCOUNT=2 namespace=lowprio create besteffort diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh index 7152813b4..a01665dab 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh @@ -1,7 +1,7 @@ cleanup() { - vm-command "kubectl delete pods --all --now" - vm-command "kubectl delete namespaces highprio lowprio --now --ignore-not-found" + delete-pods --all + delete-namespaces highprio lowprio } wait_for_waiting_status_reason() { diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index f5527f0a8..38f12802e 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -1,5 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate } diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh index 412de9d2b..929f5bf67 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh @@ -1,5 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate } @@ -176,7 +176,7 @@ verify-irq-cpus ".*ttyS0.*" "$(ctr-cpu-ids $pod ${pod}c0)" verify-irq-cpus ".*rtc0.*" "$(ctr-cpu-ids $pod ${pod}c1)" # Delete pod and check that the IRQ affinities are restored to all CPUs. -vm-command "kubectl delete pod pod0" +delete-pods $pod verify-irq-cpus ".*ttyS0.*" $ALLCPUS verify-irq-cpus ".*rtc0.*" $ALLCPUS @@ -205,7 +205,7 @@ verify-irq-cpus ".*ttyS0.*" "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod} verify-irq-cpus ".*rtc0.*" "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). -vm-command "kubectl delete pod pod1" +delete-pods $pod verify-irq-cpus ".*ttyS0.*" $ALLCPUS verify-irq-cpus ".*rtc0.*" $ALLCPUS @@ -236,7 +236,7 @@ verify-irq-cpus ".*ttyS0.*" $(ctr-cpu-ids $pod ${pod}c0) verify-irq-cpus ".*rtc0.*" "$(ids-difference "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" $(ctr-cpu-ids $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). -vm-command "kubectl delete pod pod2" +delete-pods $pod verify-irq-cpus ".*ttyS0.*" $ALLCPUS verify-irq-cpus ".*rtc0.*" $ALLCPUS @@ -421,7 +421,7 @@ ANN0=$ANN0 ANN1=$ANN1 \ verify-irq-cpus ".*ttyS0.*" "$(ctr-cpu-ids $pod ${pod}c0)" -vm-command "kubectl delete pod $pod" +delete-pods $pod unset ANN0 ANN1 @@ -448,7 +448,7 @@ ANN0=$ANN0 ANN1=$ANN1 \ verify-irq-cpus ".*rtc0.*" $ALLCPUS -vm-command "kubectl delete pod $pod" +delete-pods $pod unset ANN0 ANN1 # Create Guaranteed pod annotated to take IRQ affinity with an invalid diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh index be6445d4a..33d82d3fb 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh @@ -18,7 +18,7 @@ verify \ 'disjoint_sets(cpus["pod0c0"], cpus["pod0c1"])' \ 'disjoint_sets(packages["pod0c0"], packages["pod0c1"])' -vm-command "kubectl delete pods --all --now" +delete-pods --all helm-terminate vm-kernel-pkgs-uninstall diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh index 36e734478..f256fbf14 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh @@ -4,16 +4,15 @@ PODS=16 CONTAINERS=3 setup() { - vm-command "kubectl create namespace $TESTNS" + create-namespaces "$TESTNS" # Disable debug logging, otherwise log rotation may prevent finding remap lines. helm_config=$(DEBUG_LOGGERS="none" instantiate helm-config.yaml) helm-launch topology-aware } cleanup() { - vm-command "kubectl delete pods -n $TESTNS --all --now || :" - vm-command 'pkill -9 -f "sleep inf"' - vm-command 'pkill -9 -f "echo pod"' - vm-command "kubectl delete namespace $TESTNS --now || :" + delete-pods -n "$TESTNS" --all + kill-test-processes + delete-namespaces "$TESTNS" helm-terminate } diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test40-otel-logging/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test40-otel-logging/code.var.sh index 6a33fab09..dd06dc25d 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test40-otel-logging/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test40-otel-logging/code.var.sh @@ -2,7 +2,7 @@ OTEL_LOGS=/tmp/otel/data/otel-export.out cleanup() { always-cleanup - vm-command "kubectl delete pods --all" || : + delete-pods --all helm-terminate || : vm-command "mkdir -p /tmp/otel/data && chmod a+rw /tmp/otel/data" vm-command "rm -f $OTEL_LOGS && touch -f $OTEL_LOGS && chmod a+rw $OTEL_LOGS" From e368de2e8e7a031a88a54750e7b5dc61c0236616 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:27:27 +0300 Subject: [PATCH 03/31] e2e: add expect-launch-failure helper. Three tests expect launching the plugin to fail, and each of them spells out the same subshell, expect_error and launch_timeout incantation, plus an error for the case where the launch unexpectedly succeeds. In test13-reject-symlink this also collapses the surrounding exit status dance: the only reason for the subshell and the numeric statuses was to get the failure out of the subshell which the cache restoring trap needed. An EXIT trap in the test itself does the same, and the test code already runs in a subshell of its own. test07-maxballoons expected the failure without setting expect_error, so it relied on helm-launch calling error, and logged a confusing error of its own before reporting success. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 24 +++++++++++++ .../n4c16/test07-maxballoons/code.var.sh | 5 +-- .../test05-reserved-resources/code.var.sh | 12 +++---- .../n4c16/test13-reject-symlink/code.var.sh | 36 +++++++------------ 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index ad5de195e..676302571 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -69,6 +69,30 @@ retry-until() { # script API return 1 } +### +### Launching the plugin +### + +expect-launch-failure() { # script API + # Usage: expect-launch-failure POLICY [TIMEOUT] + # + # Expect launching POLICY to fail within TIMEOUT, 5s by default. Fail the + # test if the launch succeeds instead. + # + # Read the configuration from the helm_config variable, just like + # helm-launch does. Use this to test that the plugin refuses an invalid + # configuration: + # helm_config=$(instantiate broken-config.yaml) expect-launch-failure balloons + local policy=$1 timeout=${2:-5s} + + # helm-launch is run in a subshell, because on some failures it calls + # error, which would otherwise fail the test we expect to fail. + if ( expect_error=1 launch_timeout=$timeout helm-launch "$policy" ); then + error "launching $policy succeeded, but was expected to fail" + fi + echo "launching $policy failed as expected" +} + ### ### Cleaning up ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh index 3f4350117..2e54371f5 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh @@ -65,9 +65,6 @@ cleanup # Try starting nri-resource-policy with a configuration where MinBalloons and # MaxBalloons of the same balloon type contradict. helm-terminate -( helm_config=${TEST_DIR}/balloons-maxballoons-impossible.cfg launch_timeout=5s helm-launch balloons ) && { - error "starting nri-resource-policy succeeded, but was expected to fail due to impossible static balloons" -} -echo "starting nri-resource-policy with impossible static balloons configuration failed as expected" +helm_config=${TEST_DIR}/balloons-maxballoons-impossible.cfg expect-launch-failure balloons helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh index 134b3fdf1..7b3ebfab4 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh @@ -19,19 +19,15 @@ cleanup-kube-system helm-terminate RESERVED_CPU="cpuset:3,7,11,15" helm_config=$(instantiate helm-config.yaml) -( expect_error=1 launch_timeout=5s helm-launch topology-aware ) && error "unexpected success" || { - echo "Launch failed as expected" - get-config-node-status-error topologyawarepolicies/default || : -} +expect-launch-failure topology-aware +get-config-node-status-error topologyawarepolicies/default || : # Test launch failure, there are more reserved CPUs than available CPUs helm-terminate RESERVED_CPU='"11"' helm_config=$(instantiate helm-config.yaml) -( expect_error=1 launch_timeout=5s helm-launch topology-aware ) && error "unexpected success" || { - echo "Launch failed as expected" - get-config-node-status-error topologyawarepolicies/default || : -} +expect-launch-failure topology-aware +get-config-node-status-error topologyawarepolicies/default || : # Test that BestEffort containers are allowed to run on both Reserved # CPUs when the CPUs are on the same NUMA node. diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh index 73c4fd4ae..8116089e0 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh @@ -20,32 +20,22 @@ helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware helm-terminate topology-aware -# Turn cache into a symlink. +# Turn cache into a symlink. Restore it whatever happens, otherwise the +# symlink is left behind for the tests which run after this one. +trap restore_cache EXIT symlink_cache # Try to re-launch nri-resource-policy, check whether and how it fails. -( - trap 'restore_cache' 0 - if (expect_error=1 launch_timeout=5s helm-launch topology-aware); then - exit 1 - else - vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware" - if ! vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware | \ - grep -q 'exists, but is a symbolic link'"; then - exit 2 - else - exit 0 - fi - fi -) -status=$? +expect-launch-failure topology-aware + +vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware" +vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware | \ + grep -q 'exists, but is a symbolic link'" || + error "nri-resource-policy failed to start, but looks like for the wrong reason..." + +restore_cache +trap - EXIT helm-terminate -# Check and report test status. -case "$status" in - 1) error "ERROR: nri-resource-policy expected to reject symlinked cache, but it did not.";; - 2) error "ERROR: nri-resource-policy failed to start, but looks like for the wrong reason...";; - 0) echo "OK: nri-resource-policy properly rejected symlinked cache"; return 0;; - *) error "ERROR: test failed with unexpected status.";; -esac +echo "OK: nri-resource-policy properly rejected symlinked cache" From aa71a921bbbf9b7fb29a75ec2011818354907485 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:28:28 +0300 Subject: [PATCH 04/31] e2e: add wait-pod-gone helper. Both test18-turbo-priority and test19-pct define it. The copy in test19-pct returned the status instead of failing the test, but it has no callers, so take the one which fails the test. Poll in the VM with vm-run-until instead of from the host, so that waiting takes one ssh round trip rather than one per attempt. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 11 +++++++++++ .../balloons/n4c16/test18-turbo-priority/code.var.sh | 10 ---------- .../balloons/n4c16/test19-pct/code.var.sh | 8 -------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 676302571..85020d20f 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -69,6 +69,17 @@ retry-until() { # script API return 1 } +wait-pod-gone() { # script API + # Usage: wait-pod-gone POD [TIMEOUT] + # + # Wait until POD no longer exists, TIMEOUT seconds at most, 30 by default. + # Fail the test if the pod is still there after that. + local pod=$1 timeout=${2:-30} + vm-run-until --timeout "$timeout" "! kubectl get pod $pod -o name 2>/dev/null | grep -q ." || { + command-error "pod $pod did not disappear within ${timeout}s" + } +} + ### ### Launching the plugin ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh index 6e0e26a03..52fabe806 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh @@ -66,16 +66,6 @@ wait-enforce-grows() { } } -# wait-pod-gone [timeout=30] -# Polls until the named pod no longer exists. -wait-pod-gone() { - local pod=$1 - local timeout=${2:-30} - vm-run-until --timeout "$timeout" "! kubectl get pod $pod -o name 2>/dev/null | grep -q ." || { - command-error "pod $pod did not disappear within ${timeout}s" - } -} - # enforce-lines-since prints the enforce log lines added since the given absolute count. enforce-lines-since() { local from=$1 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index 2b667f2cf..17fd9f010 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -89,14 +89,6 @@ wait-assert-log-grew() { command-error "$msg" } -# wait-pod-gone [timeout=30] -wait-pod-gone() { - local pod=$1 - local timeout=${2:-30} - vm-run-until --timeout "$timeout" "! kubectl get pod $pod -o name 2>/dev/null | grep -q ." || return 1 - return 0 -} - # get-ext stores the given extended resource # capacity (or the string "missing") in COMMAND_OUTPUT. get-ext() { From 00a477adbe96fcfd8d9f924ca32f66d361fe1b47 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:30:55 +0300 Subject: [PATCH 05/31] e2e: add container state and creation error helpers. Three tests verify that the plugin refuses to create a container, and each of them repeats the same three steps: wait for the container to enter the CreateContainerError state, fetch its state, and grep the state for the expected error. That is twelve copies of the same block, plus two copies of a wait function which differ only in whether they address the container by name or by index. container-state accepts both, so the same helpers serve all the call sites. The regular expressions are matched with basic grep, as before. One of the patterns in test25-irq relies on parentheses being literal, so matching them as extended regular expressions would silently stop the pattern from matching what it is supposed to match. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 54 ++++++++++++ .../n4c16/test18-strict-alignment/code.var.sh | 42 +-------- .../n4c16/test19-cpuclass/code.var.sh | 48 +--------- .../n4c16/test25-irq/code.var.sh | 88 ++----------------- 4 files changed, 67 insertions(+), 165 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 85020d20f..b94fe49d8 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -41,6 +41,9 @@ retry-until() { # script API # re-read on every attempt: # retry-until --timeout 10 'vm-command "kubectl get pod $pod"' # + # SNIPPET is evaluated in the scope of this function, so it must not refer + # to variables named timeout, interval, message or elapsed. + # # This is the host side counterpart of vm-run-until. local timeout=30 interval=1 message="" elapsed=0 while [ "${1#--}" != "$1" ]; do @@ -69,6 +72,57 @@ retry-until() { # script API return 1 } +container-state() { # script API + # Usage: container-state POD CONTAINER + # + # Print the state object of CONTAINER of POD, and store it in + # COMMAND_OUTPUT. CONTAINER is a container name, the index of a container + # in the pod, or empty for all containers of the pod. + local pod=$1 ctr=$2 jqsel + if [ -z "$ctr" ]; then + jqsel='.[]' + elif [[ "$ctr" =~ ^[0-9]+$ ]]; then + jqsel=".[$ctr]" + else + jqsel="map(select(.name == \"$ctr\")) | .[0]" + fi + vm-command "kubectl get pod $pod -ojson | \ + jq '.status.containerStatuses | $jqsel | .state'" +} + +wait-container-waiting-reason() { # script API + # Usage: wait-container-waiting-reason POD CONTAINER REASON [TIMEOUT] + # + # Wait until the waiting reason of CONTAINER of POD becomes REASON, + # TIMEOUT seconds at most, 5 by default. CONTAINER is passed to + # container-state, so it can also be a container index or empty. + # Fail the test on timeout. + local pod=$1 ctr=$2 reason=$3 timeout=${4:-5} + retry-until --timeout "$timeout" \ + 'container-state "$pod" "$ctr" && grep -q "$reason" <<< "$COMMAND_OUTPUT"' || { + error "container ${ctr:-*} of pod $pod did not enter the $reason state" + } +} + +verify-container-error() { # script API + # Usage: verify-container-error POD CONTAINER REGEXP [TIMEOUT] + # + # Verify that creating CONTAINER of POD failed with an error matching + # REGEXP. Wait TIMEOUT seconds, 5 by default, for the container to enter + # the CreateContainerError state, then require REGEXP to match its state. + # + # Create the pod with wait="" to keep the framework from waiting for a + # pod which is never going to become ready: + # wait="" CONTCOUNT=1 create guaranteed + # verify-container-error pod0 pod0c0 "invalid IRQ affinity" + local pod=$1 ctr=$2 regexp=$3 timeout=${4:-5} + + wait-container-waiting-reason "$pod" "$ctr" CreateContainerError "$timeout" + container-state "$pod" "$ctr" + grep -q "$regexp" <<< "$COMMAND_OUTPUT" || + error "expected an error matching \"$regexp\" from creating container ${ctr:-*} of pod $pod" +} + wait-pod-gone() { # script API # Usage: wait-pod-gone POD [TIMEOUT] # diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh index a01665dab..40edc8f6b 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test18-strict-alignment/code.var.sh @@ -4,21 +4,6 @@ cleanup() { delete-namespaces highprio lowprio } -wait_for_waiting_status_reason() { - local pod=$1 ctridx=$2 status=$3 - local maxtry=5 retry=0 - - while [ $retry -lt $maxtry ]; do - echo "Waiting for $pod/ctr$ctridx to enter status $status..." - vm-command "kubectl get pod $pod -o jsonpath='{.status.containerStatuses[$ctridx].state}' | \ - jq '.waiting.reason'" - grep -q $status <<< "$COMMAND_OUTPUT" && return 0 - let retry=$retry+1 - sleep 1 - done - return 1 -} - cleanup helm-terminate helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware @@ -64,14 +49,7 @@ vm-command "kubectl wait --timeout=5s pod pod3 --for=$PodReadyCond" || { error "failed to wait for pod3 to start containerd" } -wait_for_waiting_status_reason pod3 0 CreateContainerError || { - error "failed to wait for pod3/ctr0 to reach CreateContainerError state" -} - -vm-command "kubectl get pod pod3 -o jsonpath='{.status.containerStatuses[0].state}' | \ - grep -q \"fail strict hint\"" || { - error "pod3c0 unexpectedly passed strict topology hint check" -} +verify-container-error pod3 0 "fail strict hint" # Try to create a container with a strict test hints for NUMA node 2 and # required isolated CPUs. This one would fit but there are not isolated @@ -86,14 +64,7 @@ vm-command "kubectl wait --timeout=5s pod pod4 --for=$PodReadyCond" || { error "failed to wait for pod4 to start containerd" } -wait_for_waiting_status_reason pod4 0 CreateContainerError || { - error "failed to wait for pod4/ctr0 to reach CreateContainerError state" -} - -vm-command "kubectl get pod pod4 -o jsonpath='{.status.containerStatuses[0].state}' | \ - grep -q \"isolated CPUs\"" || { - error "pod4c0 unexpectedly passed strict isolated CPU requirement check" -} +verify-container-error pod4 0 "isolated CPUs" # Create a container with a strict test hint for NUMA node 2 and preference # for isolated CPUs. This one should fit and succeed because the unfulfilled @@ -124,14 +95,7 @@ vm-command "kubectl wait --timeout=5s pod pod6 --for=$PodReadyCond" || { error "failed to wait for pod6 to start containerd" } -wait_for_waiting_status_reason pod6 0 CreateContainerError || { - error "failed to wait for pod6/ctr0 to reach CreateContainerError state" -} - -vm-command "kubectl get pod pod6 -o jsonpath='{.status.containerStatuses[0].state}' | \ - grep -q \"isolated CPUs\"" || { - error "pod6c0 unexpectedly passed strict isolated CPU requirement check" -} +verify-container-error pod6 0 "isolated CPUs" # Now recreate pod5 test but with more complex effective annotation. # Create a container with a strict test hint for NUMA node 2 and preference diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 38f12802e..6f708abfc 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -119,27 +119,6 @@ print(" ".join(str(x) for x in sorted(r))) ' "$cpus" } -# wait-pod-waiting-reason [] -# Wait until the pods waiting reason becomes the given one. -wait-pod-waiting-reason() { - local pod=$1 reason=$2 timeout=${3:-5} - local cnt=0 - - while true; do - vm-command "kubectl get pod $pod -o json | \ - jq '.status.containerStatuses[].state.waiting.reason'" - grep -q $reason <<<$COMMAND_OUTPUT && break - - if [ $cnt -lt $timeout ]; then - let cnt=$cnt+1 - sleep 1 - continue - fi - - error "Failed to wait for CreateContainerError of $pod" - done -} - OVERRIDE_SYS_CPUFREQ='[{"cpus": "0-15", "base": 2900000, "min": 800000, "max": 3800000}]' OVERRIDE_SST='{"supported": true, "clos_count": 4, "packages": [{"id": 0, "cpus": "0-7", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}, {"id": 1, "cpus": "8-15", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}]}' OVERRIDE_SST_STATE_DIR="/tmp/nri-pct-mock" @@ -249,14 +228,7 @@ pod=pod2 ANN0="cpu-class.resource-policy.nri.io/container.${pod}c0: class1" \ wait="" CONTCOUNT=1 create burstable -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "CPU class .* invalid for non-Guaranteed QoS class" <<< $COMMAND_OUTPUT || - error "Missing QoS-based cpuclass denial error for $ctr" +verify-container-error $pod ${pod}c0 "CPU class .* invalid for non-Guaranteed QoS class" # BestEffort QoS Class @@ -264,14 +236,7 @@ pod=pod3 ANN0="cpu-class.resource-policy.nri.io/container.${pod}c0: class1" \ wait="" CONTCOUNT=1 create besteffort -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "CPU class .* invalid for non-Guaranteed QoS class" <<< $COMMAND_OUTPUT || - error "Missing QoS-based cpuclass denial error for $ctr" +verify-container-error $pod ${pod}c0 "CPU class .* invalid for non-Guaranteed QoS class" # Guaranteed QoS Class Without Exclusive CPU allocation @@ -279,14 +244,7 @@ pod=pod4 ANN0="cpu-class.resource-policy.nri.io/container.${pod}c0: class1" \ wait="" CPU=250m CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "CPU class .* invalid without exclusive CPUs" <<< $COMMAND_OUTPUT || - error "Missing QoS-based cpuclass denial error for $ctr" +verify-container-error $pod ${pod}c0 "CPU class .* invalid without exclusive CPUs" cleanup diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh index 929f5bf67..323d89b1c 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh @@ -121,27 +121,6 @@ verify-irq-cpus() { error "IRQ $irqnum affinity: expected CPUs '$expected', got '$got'" } -# wait-pod-waiting-reason [] -# Wait until the pods waiting reason becomes the given one. -wait-pod-waiting-reason() { - local pod=$1 reason=$2 timeout=${3:-5} - local cnt=0 - - while true; do - vm-command "kubectl get pod $pod -o json | \ - jq '.status.containerStatuses[].state.waiting.reason'" - grep -q $reason <<<$COMMAND_OUTPUT && break - - if [ $cnt -lt $timeout ]; then - let cnt=$cnt+1 - sleep 1 - continue - fi - - error "Failed to wait for CreateContainerError of $pod" - done -} - cleanup DEBUG_LOGGERS="irq" @@ -254,14 +233,7 @@ EOF ANN0=$ANN0 \ wait="" CPU=1 CONTCOUNT=1 create besteffort -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "invalid IRQ affinity, QoS class .* is not Guaranteed" <<< $COMMAND_OUTPUT || - error "Missing QoS-based IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "invalid IRQ affinity, QoS class .* is not Guaranteed" # Create Burstable pod, try to annotate container for IRQ affinity. Should fail. @@ -274,14 +246,7 @@ EOF ANN0=$ANN0 \ wait="" CPU=1 CONTCOUNT=1 create burstable -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "invalid IRQ affinity, QoS class .* is not Guaranteed" <<< $COMMAND_OUTPUT || - error "Missing QoS-based IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "invalid IRQ affinity, QoS class .* is not Guaranteed" unset ANN0 @@ -297,14 +262,7 @@ EOF ANN0=$ANN0 \ wait="" CPU=250m CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "IRQ affinity .* invalid without exclusive CPUs" <<< $COMMAND_OUTPUT || - error "Missing shared CPU-based IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "IRQ affinity .* invalid without exclusive CPUs" unset ANN0 @@ -320,14 +278,7 @@ EOF ANN0=$ANN0 \ wait="" CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "invalid IRQ affinity .*: .*" <<< $COMMAND_OUTPUT || - error "Missing unparsable IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "invalid IRQ affinity .*: .*" unset ANN0 @@ -345,14 +296,7 @@ EOF ANN0=$ANN0 \ wait="" CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "invalid IRQ affinity mode .*xyzzy.* (valid modes: .*)" <<< $COMMAND_OUTPUT || - error "Missing invalid mode based IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "invalid IRQ affinity mode .*xyzzy.* (valid modes: .*)" cleanup unset ANN0 @@ -378,14 +322,7 @@ EOF ANN0=$ANN0 \ wait="" CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "denied interrupt: .* denied but matched by user pattern .*" <<< $COMMAND_OUTPUT || - error "Missing IRQ affinity denial error for $ctr" +verify-container-error $pod ${pod}c0 "denied interrupt: .* denied but matched by user pattern .*" cleanup unset ANN0 @@ -473,18 +410,7 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ wait="" CONTCOUNT=1 create guaranteed -wait-pod-waiting-reason $pod CreateContainerError - -ctr=${pod}c0 -vm-command "kubectl get pods $pod -ojson | \ - jq '.status.containerStatuses | map(select(.name == \"$ctr\")) | .[0].state'" - -grep -q "invalid IRQ affinity devices pattern" <<< $COMMAND_OUTPUT || - error "Missing invalid affinity devices pattern error for $ctr" - - -verify-irq-cpus ".*rtc0.*" $ALLCPUS - +verify-container-error $pod ${pod}c0 "invalid IRQ affinity devices pattern" cleanup unset ANN0 ANN1 From e2ab3fe3ac86126afaed908371ceb0777a30086e Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:34:25 +0300 Subject: [PATCH 06/31] e2e: add plugin log retrieval helpers. Eight tests read the log of the plugin daemonset, each of them spelling out the kubectl command with a hardcoded daemonset name. plugin-log derives the daemonset from $POLICY the same way helm-launch does, and returns the status of matching the pattern, so that callers can keep reporting a missing log line themselves. It also retries while the log is not available yet, which happens right after the plugin has restarted. This drops pull-logs from test31-duplicate-disambiguation. Its retry counter was never incremented, because it added one to $ctn instead of $cnt, so it kept retrying until the logs became available rather than giving up after five attempts. The retry it really needs, that is, re-running the whole scenario when no remapping was triggered, is in check-logs and stays as it is. The tests which also assert something about the log are converted separately. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 75 +++++++++++++++++++ .../n4c16/test24-podresources/code.var.sh | 4 +- .../n4c16/test13-reject-symlink/code.var.sh | 5 +- .../code.var.sh | 22 +----- 4 files changed, 81 insertions(+), 25 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index b94fe49d8..d9ddf5579 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,81 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### Reading the log of the plugin +### + +plugin-daemonset() { # script API + # Usage: plugin-daemonset [PLUGIN] + # + # Print the name of the DaemonSet of PLUGIN, $POLICY by default. + # + # This mirrors the daemonset_name defaults of helm-launch. + local plugin=${1:-$POLICY} + case "$plugin" in + *topology*aware*) echo nri-resource-policy-topology-aware;; + *balloons*) echo nri-resource-policy-balloons;; + *memory-policy*) echo nri-memory-policy;; + *memtierd*) echo nri-memtierd;; + *) error "plugin-daemonset: unknown plugin \"$plugin\"";; + esac +} + +plugin-log() { # script API + # Usage: plugin-log [--plugin PLUGIN] [--tail LINES] [--ignore-case] [PATTERN] + # + # Print the log of the DaemonSet of PLUGIN, $POLICY by default, and store + # it in COMMAND_OUTPUT. If PATTERN is given, print only the lines matching + # it as an extended regular expression, case-insensitively if + # --ignore-case is given. If LINES is given, print only the last LINES of + # the matching lines. + # + # Return non-zero if PATTERN did not match anything, so that the caller + # can report a missing log line: + # plugin-log 'associated cpus .* to CLOS 0' || command-error "no CLOS 0" + # + # Retry while the log of the plugin is not available. That happens for + # instance right after the plugin has restarted. + + # What kubectl says while the log of a container is not readable yet. + local unavailable="unable to retrieve container logs for" + local plugin=$POLICY lines="" pattern="" grepopts="-E" cmd + while [ "${1#--}" != "$1" ]; do + case "$1" in + --plugin) plugin="$2"; shift 2;; + --tail) lines="$2"; shift 2;; + --ignore-case) grepopts="$grepopts -i"; shift;; + --) shift; break;; + *) error "plugin-log: unknown option \"$1\"";; + esac + done + pattern="$1" + + cmd="kubectl -n kube-system logs ds/$(plugin-daemonset "$plugin") 2>&1" + if [ -n "$pattern" ]; then + # Keep the transient availability error visible to the retry below. + # Filtering it out would leave nothing for the retry to notice, and it + # would give up after a single attempt. + cmd="$cmd | grep $grepopts -e '$pattern' -e '$unavailable'" + fi + if [ -n "$lines" ]; then + cmd="$cmd | tail -n $lines" + fi + + retry-until --timeout 15 --interval 3 \ + 'vm-command "$cmd"; ! grep -q "$unavailable" <<< "$COMMAND_OUTPUT"' || : + + # The log never became readable. Say so rather than reporting the status of + # matching the pattern: with the pattern above that status is the status of + # matching the availability error itself, that is, success. + if grep -q "$unavailable" <<< "$COMMAND_OUTPUT"; then + return 1 + fi + + # Report whether PATTERN matched, not whether the log became available. + return "$COMMAND_STATUS" +} + ### ### Cleaning up ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh index d4d783ddf..829a99c35 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh @@ -21,7 +21,7 @@ cleanup() { verify-podres-locality() { local resource=$1 shift - vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | grep 'pod-resource device \"$resource\"'" >/dev/null \ + plugin-log "pod-resource device \"$resource\"" > /dev/null \ || command-error "no device-locality log lines for resource $resource" local log="$COMMAND_OUTPUT" local ctr @@ -146,7 +146,7 @@ verify 'disjoint_sets(nodes["pod1c0"], nodes["pod1c1"], nodes["pod1c2"], nodes[" # Sanity check (cf. test19-pct): the hp-near-tpu balloons use the # pct-hp cpuClass, so their CPUs must have been associated to the PCT # high-priority CLOS 0. -vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | grep -E 'associated cpus .* to CLOS 0'" \ +plugin-log 'associated cpus .* to CLOS 0' \ || command-error "hp-near-tpu balloon CPUs were not associated to PCT HP CLOS 0" vm-command "kubectl delete pods --all --now" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh index 8116089e0..955577476 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test13-reject-symlink/code.var.sh @@ -28,9 +28,8 @@ symlink_cache # Try to re-launch nri-resource-policy, check whether and how it fails. expect-launch-failure topology-aware -vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware" -vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware | \ - grep -q 'exists, but is a symbolic link'" || +plugin-log +plugin-log 'exists, but is a symbolic link' || error "nri-resource-policy failed to start, but looks like for the wrong reason..." restore_cache diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh index f256fbf14..4f923fc8c 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test31-duplicate-disambiguation/code.var.sh @@ -55,7 +55,7 @@ wait-containers-restart() { while ! [[ "$statuses" == "Running" ]]; do sleep 5 - if vm-command "kubectl logs -n kube-system ds/nri-resource-policy-topology-aware 2>&1 | grep -iE 'remap|keeping|duplicate'"; then + if plugin-log --ignore-case 'remap|keeping|duplicate'; then break fi vm-command "kubectl get pods -A --no-headers=true | tr -s '\t' ' '| cut -d ' ' -f4 | sort -u" @@ -93,26 +93,8 @@ check-no-duplicate-allocations() { fi } -pull-logs() { - local cnt=0 - while [ $cnt -lt 5 ]; do - vm-command "kubectl logs -n kube-system ds/nri-resource-policy-topology-aware 2>&1 | grep -iE 'remap|keeping|duplicate|unable'" - if grep -q 'unable to retrieve container logs for' <<< $COMMAND_OUTPUT; then - echo "Unable to retrieve policy logs, retrying..." - sleep 3 - let cnt=$ctn+1 - else - return 0 - fi - done - return 1 -} - check-logs() { - if ! pull-logs; then - echo "Failed to pull policy logs..." - return 1 - fi + plugin-log --ignore-case 'remap|keeping|duplicate' || : if ! check-transient-duplicates-present; then return 1 From 0add141acee5bc4863b1d2a1def45a73ce30a080 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:40:54 +0300 Subject: [PATCH 07/31] e2e: add plugin log assertion helpers. test19-pct, test19-cpuclass and test18-turbo-priority each define their own function for fetching the tail of the log lines of the subsystem they test, and the first two also define near-identical assertions on top of it. The only real difference between the three is which log lines they are interested in, so keep that in the tests as plugin_log_filter and share everything else. The signatures of the assertions are unchanged, so their 27 call sites stay as they are. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 60 ++++++++++++++++ .../n4c16/test18-turbo-priority/code.var.sh | 12 ++-- .../balloons/n4c16/test19-pct/code.var.sh | 68 ++----------------- .../balloons/n4c16/test25-irq/code.var.sh | 4 +- .../n4c16/test19-cpuclass/code.var.sh | 33 +-------- 5 files changed, 74 insertions(+), 103 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index d9ddf5579..ce0a26cfd 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -233,6 +233,66 @@ plugin-log() { # script API return "$COMMAND_STATUS" } +plugin-log-tail() { # script API + # Usage: plugin-log-tail [LINES] + # + # Print the last LINES lines of the plugin log which match the extended + # regular expression in $plugin_log_filter, all lines if the variable is + # unset. LINES defaults to $plugin_log_tail_lines, or 500. + # + # This is what the log assertions below look at. Set plugin_log_filter in + # a test to restrict them to the log of the subsystem under test: + # plugin_log_filter='pct(:| mock:)' + plugin-log --tail "${1:-${plugin_log_tail_lines:-500}}" "${plugin_log_filter:-}" || : +} + +assert-log-contains() { # script API + # Usage: assert-log-contains REGEXP [MESSAGE] + # + # Fail the test unless REGEXP, an extended regular expression, matches the + # plugin log. See plugin-log-tail for which part of the log is inspected. + local pattern=$1 msg=${2:-"expected log line missing"} + plugin-log-tail + grep -E -q "$pattern" <<< "$COMMAND_OUTPUT" || command-error "$msg (pattern: $pattern)" +} + +assert-log-not-contains() { # script API + # Usage: assert-log-not-contains REGEXP [MESSAGE] + # + # Fail the test if REGEXP matches the plugin log. + local pattern=$1 msg=${2:-"unexpected log line"} + plugin-log-tail + if grep -E -q "$pattern" <<< "$COMMAND_OUTPUT"; then + command-error "$msg (unexpected pattern: $pattern)" + fi +} + +wait-assert-log-contains() { # script API + # Usage: wait-assert-log-contains REGEXP [MESSAGE] [TIMEOUT] + # + # Wait until REGEXP matches the plugin log, TIMEOUT seconds at most, 5 by + # default. Fail the test on timeout, reporting the log which was inspected. + local pattern=$1 msg=${2:-"expected log line missing"} tmo=${3:-5} + retry-until --timeout "$tmo" \ + 'plugin-log-tail; grep -E -q "$pattern" <<< "$COMMAND_OUTPUT"' || + assert-log-contains "$pattern" "$msg" +} + +wait-assert-log-grew() { # script API + # Usage: wait-assert-log-grew REGEXP COUNT [MESSAGE] [TIMEOUT] + # + # Wait until the plugin log has more than COUNT lines matching REGEXP, + # TIMEOUT seconds at most, 5 by default. Fail the test on timeout. + # + # Use this instead of wait-assert-log-contains when REGEXP already matched + # something in an earlier phase of the test, and the point is that a new + # line shows up. + local pattern=$1 count=$2 msg=${3:-"expected new log lines"} tmo=${4:-5} + retry-until --timeout "$tmo" \ + 'plugin-log-tail; [ "$(grep -c -E "$pattern" <<< "$COMMAND_OUTPUT")" -gt "$count" ]' || + command-error "$msg (pattern: $pattern, expected more than $count lines)" +} + ### ### Cleaning up ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh index 52fabe806..52adbf2d1 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh @@ -15,11 +15,9 @@ helm-terminate helm_config=$TEST_DIR/balloons-turbo.cfg helm-launch balloons -# turbo-log fetches the latest turbo recalculation log lines -turbo-log() { - local last_n=${1:-20} - vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | grep -E 'turbo:|cpuClass' | tail -n $last_n" -} +# Restrict the log assertions to the turbo recalculation log lines. +plugin_log_filter='turbo:|cpuClass' +plugin_log_tail_lines=20 # verify-turbo-winner checks that the given class is logged as a turbo winner # with the expected maxFreq, within the last N turbo log lines. @@ -28,7 +26,7 @@ verify-turbo-winner() { local expected_max_freq=$2 local last_n=${3:-20} echo "verify turbo winner: class=$class maxFreq=$expected_max_freq" - turbo-log $last_n + plugin-log-tail $last_n grep "class \"$class\"" <<< "$COMMAND_OUTPUT" | grep "winner=true" | tail -n 1 | grep -q "maxFreq=$expected_max_freq" || { command-error "expected class $class as turbo winner with maxFreq=$expected_max_freq" } @@ -41,7 +39,7 @@ verify-turbo-loser() { local expected_max_freq=$2 local last_n=${3:-20} echo "verify turbo loser: class=$class maxFreq=$expected_max_freq" - turbo-log $last_n + plugin-log-tail $last_n grep "class \"$class\"" <<< "$COMMAND_OUTPUT" | grep "winner=false" | tail -n 1 | grep -q "maxFreq=$expected_max_freq" || { command-error "expected class $class as turbo loser with maxFreq=$expected_max_freq" } diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index 17fd9f010..4eecac2f2 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -28,66 +28,8 @@ helm-terminate -# pct-log fetches the latest PCT-related log lines. -pct-log() { - local last_n=${1:-200} - vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | grep -E 'pct(:| mock:)' | tail -n $last_n" -} - -# assert-log-contains -assert-log-contains() { - local pat=$1 - local msg=$2 - pct-log 500 - grep -E -q "$pat" <<< "$COMMAND_OUTPUT" || command-error "$msg (pattern: $pat)" -} - -# assert-log-not-contains -assert-log-not-contains() { - local pat=$1 - local msg=$2 - pct-log 500 - if grep -E -q "$pat" <<< "$COMMAND_OUTPUT"; then - command-error "$msg (unexpected pattern: $pat)" - fi -} - -# wait-assert-log-contains [timeout=5] -# Polls the pct log every 1s until matches or -# seconds pass. On timeout, defers to assert-log-contains so the -# resulting command-error carries the captured log output. -wait-assert-log-contains() { - local pat=$1 - local msg=$2 - local timeout=${3:-5} - local elapsed=0 - while [ "$elapsed" -lt "$timeout" ]; do - pct-log 500 - grep -E -q "$pat" <<< "$COMMAND_OUTPUT" && return 0 - sleep 1 - elapsed=$((elapsed + 1)) - done - assert-log-contains "$pat" "$msg" -} - -# wait-assert-log-grew [timeout=5] -# Like wait-assert-log-contains but for "did a fresh line appear?" -# cases where the pattern already exists from an earlier phase. -wait-assert-log-grew() { - local pat=$1 - local prev=$2 - local msg=$3 - local timeout=${4:-5} - local elapsed=0 cur - while [ "$elapsed" -lt "$timeout" ]; do - pct-log 500 - cur=$(grep -c -E "$pat" <<< "$COMMAND_OUTPUT") - [ "$cur" -gt "$prev" ] && return 0 - sleep 1 - elapsed=$((elapsed + 1)) - done - command-error "$msg" -} +# Restrict the log assertions to the PCT-related log lines. +plugin_log_filter='pct(:| mock:)' # get-ext stores the given extended resource # capacity (or the string "missing") in COMMAND_OUTPUT. @@ -209,7 +151,7 @@ verify 'packages["pod2c0"] != packages["pod0c0"]' # scarce; what matters is that the resize happens AND the new # CPUs are programmed to the correct CLOS (the cpuclass-driven # behavior under test). -pct-log 500 +plugin-log-tail 500 prev_to_clos0=$(grep -c 'to CLOS 0' <<< "$COMMAND_OUTPUT") CPUREQ=1 CPULIM=1 MEMREQ=10M MEMLIM=10M \ EXTREQ="cpuclass.balloons.nri.io/pct-hp: \"1\"" \ @@ -242,14 +184,14 @@ report allowed # the idleCpuClass "default-class" has no PCT plan, and managed # mode must NOT silently park idle CPUs on the HP CLOS 0 (which # would consume limited Priority Core Turbo capacity). -pct-log 500 +plugin-log-tail 500 prev_to_clos3=$(grep -c 'to CLOS 3' <<< "$COMMAND_OUTPUT") vm-command "kubectl delete pod pod1 --now" wait-assert-log-grew 'to CLOS 3' "$prev_to_clos3" "deleting LP pod did not reassociate its CPUs to LP fallback CLOS 3" # Now delete the rest -- all freed CPUs end up on the LP # fallback CLOS 3 for the same reason. -pct-log 500 +plugin-log-tail 500 prev_to_clos3=$(grep -c 'to CLOS 3' <<< "$COMMAND_OUTPUT") vm-command "kubectl delete pods --all --now" wait-assert-log-grew 'to CLOS 3' "$prev_to_clos3" "after deleting remaining pods CPUs were not reassociated to LP fallback CLOS 3" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index 67180769f..522190869 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -167,8 +167,8 @@ helm-terminate # Test restricting interrupts control and verify that it takes effect. expect_error=1 helm_config=$TEST_DIR/balloons-denied-irq-claim.cfg helm-launch balloons -vm-run-until --timeout 10 "kubectl -n kube-system logs ds/nri-resource-policy-balloons 2>/dev/null | grep -q 'denied interrupt: .* denied but matched by user pattern .*'" || \ - command-error "expected error of IRQ claim referencing denied IRQ not reported" +wait-assert-log-contains 'denied interrupt: .* denied but matched by user pattern .*' \ + "expected error of IRQ claim referencing denied IRQ not reported" 10 helm-terminate || true cleanup diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 6f708abfc..1ce3a8592 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -3,37 +3,8 @@ cleanup() { helm-terminate } -# fetch-log fetches the latest log lines matching a given pattern. -fetch-log() { - local last_n=${1:-200} pattern=${2:-' *cpuclass *'} - vm-command "kubectl -n kube-system logs ds/nri-resource-policy-topology-aware | grep -E \"$pattern\" | tail -n $last_n" -} - -# assert-log-contains -assert-log-contains() { - local pat=$1 - local msg=$2 - fetch-log 500 - grep -E -q "$pat" <<< "$COMMAND_OUTPUT" || command-error "$msg (pattern: $pat)" -} - -# wait-assert-log-contains [timeout=5] -# Polls the policy log every 1s until matches or -# seconds pass. On timeout, defers to assert-log-contains so the -# resulting command-error carries the captured log output. -wait-assert-log-contains() { - local pat=$1 - local msg=$2 - local timeout=${3:-5} - local elapsed=0 - while [ "$elapsed" -lt "$timeout" ]; do - fetch-log 500 - grep -E -q "$pat" <<< "$COMMAND_OUTPUT" && return 0 - sleep 1 - elapsed=$((elapsed + 1)) - done - assert-log-contains "$pat" "$msg" -} +# Restrict the log assertions to the cpuclass-related log lines. +plugin_log_filter=' *cpuclass *' # get-ext stores the given extended resource # capacity (or the string "missing") in COMMAND_OUTPUT. From e0e1e9991cacd1c65b616d5e0c85c2092679d878 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:48:12 +0300 Subject: [PATCH 08/31] e2e: add node extended resource helpers. test19-pct and test19-cpuclass have identical functions for reading and waiting for an extended resource of the node, and test24-podresources waits for one by grepping the output of kubectl describe node in a hand-rolled 60 round loop. Reading the value with jq also makes the comparison in test24-podresources exact: it used to accept any allocatable amount of tech.com/tpu containing a 4. test19-pct keeps thin wrappers of its own, so that the name of the resource it polls stays in one place. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 43 +++++++++++++++++++ .../balloons/n4c16/test19-pct/code.var.sh | 29 +++---------- .../n4c16/test24-podresources/code.var.sh | 13 ++---- .../n4c16/test19-cpuclass/code.var.sh | 31 +------------ 4 files changed, 55 insertions(+), 61 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index ce0a26cfd..412f4ab29 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,49 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### Extended resources of the node +### + +get-node-resource() { # script API + # Usage: get-node-resource [--allocatable] NAME + # + # Print the capacity, or with --allocatable the allocatable amount, of + # extended resource NAME on the test node, and store it in COMMAND_OUTPUT. + # Print "missing" if the node does not have the resource at all. + local field=capacity + while [ "${1#--}" != "$1" ]; do + case "$1" in + --capacity) field=capacity; shift;; + --allocatable) field=allocatable; shift;; + *) error "get-node-resource: unknown option \"$1\"";; + esac + done + vm-command "kubectl get nodes -o json | jq -r '.items[] | (.status.$field[\"$1\"] // \"missing\")'" +} + +wait-node-resource() { # script API + # Usage: wait-node-resource [--allocatable] [--timeout SECS] [--interval SECS] NAME VALUE [MESSAGE] + # + # Wait until extended resource NAME on the test node equals VALUE, which + # can also be the string "missing". Give up after SECS seconds, 30 by + # default, checking every SECS seconds, 2 by default. Fail the test with + # MESSAGE on timeout. + local fieldopt="" tmo=30 ival=2 + while [ "${1#--}" != "$1" ]; do + case "$1" in + --timeout) tmo="$2"; shift 2;; + --interval) ival="$2"; shift 2;; + *) fieldopt="$1"; shift;; + esac + done + local name=$1 value=$2 msg=${3:-"unexpected amount of $1 on the node"} + retry-until --timeout "$tmo" --interval "$ival" \ + 'get-node-resource $fieldopt "$name" && [ "$COMMAND_OUTPUT" == "$value" ]' || { + command-error "$msg (expected '$value', got '$COMMAND_OUTPUT')" + } +} + ### ### Reading the log of the plugin ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index 4eecac2f2..b16e9708b 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -31,32 +31,15 @@ helm-terminate # Restrict the log assertions to the PCT-related log lines. plugin_log_filter='pct(:| mock:)' -# get-ext stores the given extended resource -# capacity (or the string "missing") in COMMAND_OUTPUT. -get-ext() { - local name=$1 - vm-command "kubectl get nodes -o json | jq -r '.items[] | (.status.capacity[\"$name\"] // \"missing\")'" -} +# The extended resource which the pct-hp cpuClass publishes. +ext_hp="cpuclass.balloons.nri.io/pct-hp" -# get-ext-hp stores the pct-hp extended resource capacity (or the -# string "missing") in COMMAND_OUTPUT. get-ext-hp() { - get-ext "cpuclass.balloons.nri.io/pct-hp" + get-node-resource "$ext_hp" } -# wait-ext-hp [timeout=30] [interval=2] -# Polls until the pct-hp extended resource equals (a number -# or the string "missing"), or fails with on timeout. wait-ext-hp() { - local want=$1 msg=$2 timeout=${3:-30} interval=${4:-2} elapsed=0 - while [ "$elapsed" -lt "$timeout" ]; do - get-ext-hp - [ "$COMMAND_OUTPUT" == "$want" ] && return 0 - sleep "$interval" - elapsed=$((elapsed + interval)) - done - get-ext-hp - command-error "$msg (expected '$want', got '$COMMAND_OUTPUT')" + wait-node-resource "$ext_hp" "$@" } # helm-upgrade performs an in-process reconfiguration by @@ -246,7 +229,7 @@ get-ext-hp vm-command "kubectl patch node $node_name --subresource=status --type merge \ -p '{\"status\":{\"capacity\":{\"example.com/not-owned\":\"7\"}}}'" || command-error "failed to inject unrelated non-nri.io extended resource" -get-ext "example.com/not-owned" +get-node-resource "example.com/not-owned" [ "$COMMAND_OUTPUT" == "7" ] || \ command-error "expected injected non-nri.io extended resource (7) to be visible, got '$COMMAND_OUTPUT'" @@ -260,7 +243,7 @@ wait-ext-hp missing "reconciliation on launch did not remove the orphan HP exten # The unrelated non-nri.io resource must NOT have been removed: the # agent refuses to touch anything outside the *.nri.io/* domain. -get-ext "example.com/not-owned" +get-node-resource "example.com/not-owned" [ "$COMMAND_OUTPUT" == "7" ] || \ command-error "reconciliation removed or altered a non-nri.io extended resource (expected '7', got '$COMMAND_OUTPUT')" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh index 829a99c35..d63bcda7e 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh @@ -91,15 +91,10 @@ sleep 1 # Wait until both fake device plugins have registered and their # devices are Allocatable. -rounds=0 -while vm-command "kubectl describe node \$(hostname) | grep -E 'Capacity|Alloc|telco.com|tech.com'"; do - ( grep -A2 Allocatable <<< "$COMMAND_OUTPUT" | grep -qE 'tech.com/tpu:.*4' ) && \ - ( grep -A2 Allocatable <<< "$COMMAND_OUTPUT" | grep -qE 'telco.com/nic:.*2' ) && \ - break - rounds=$(( rounds + 1 )) - (( rounds > 60 )) && error "waiting for fake-device-plugin resources timed out" - sleep 1 -done +wait-node-resource --allocatable --timeout 60 --interval 1 tech.com/tpu 4 \ + "fake TPU device plugin did not publish its devices" +wait-node-resource --allocatable --timeout 60 --interval 1 telco.com/nic 2 \ + "fake NIC device plugin did not publish its devices" # burstable containers CPUREQ=2 CPULIM=4 MEMREQ=10M MEMLIM=50M \ diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 1ce3a8592..c8be809de 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -6,34 +6,6 @@ cleanup() { # Restrict the log assertions to the cpuclass-related log lines. plugin_log_filter=' *cpuclass *' -# get-ext stores the given extended resource -# capacity (or the string "missing") in COMMAND_OUTPUT. -get-ext() { - local name=$1 - vm-command "kubectl get nodes -o json | jq -r '.items[] | (.status.capacity[\"$name\"] // \"missing\")'" -} - -# get-ext-exclusive stores the exclusive CPU class extended resource capacity (or the -# string "missing") in COMMAND_OUTPUT. -get-ext-exclusive() { - get-ext "cpuclass.resource-policy.nri.io/exclusive" -} - -# wait-ext-exclusive [timeout=30] [interval=2] -# Polls until the exclusive CPU class extended resource equals (a number -# or the string "missing"), or fails with on timeout. -wait-ext-exclusive() { - local want=$1 msg=$2 timeout=${3:-30} interval=${4:-2} elapsed=0 - while [ "$elapsed" -lt "$timeout" ]; do - get-ext-exclusive - [ "$COMMAND_OUTPUT" == "$want" ] && return 0 - sleep "$interval" - elapsed=$((elapsed + interval)) - done - get-ext-exclusive - command-error "$msg (expected '$want', got '$COMMAND_OUTPUT')" -} - # ctr-cpu-ids # Inspect the container of the given pod and report the cpuset it is pinned to. ctr-cpu-ids() { @@ -123,7 +95,8 @@ wait-assert-log-contains 'PrepareManagedMode done' "managed mode startup missing wait-assert-log-contains 'ConfigureClos.*ClosID:0 MinFreq:3800000 MaxFreq:3800000' "HP CLOS 0 not programmed with MinFreq=MaxFreq=turbo (3800000)" wait-assert-log-contains 'ConfigureClos.*ClosID:3 MinFreq:800000 MaxFreq:2900000' "LP CLOS 3 not programmed with MinFreq=min (800000) MaxFreq=base (2900000)" wait-assert-log-contains 'EnableCP done' "EnableCP missing" -wait-ext-exclusive 4 "expected 4 PCT HP CPUs published as extended resources" +wait-node-resource cpuclass.resource-policy.nri.io/exclusive 4 \ + "expected 4 PCT HP CPUs published as extended resources" # # Reserved pool CPU class assignment From 886d917e077b874e3e4e4c57475e37a9eaf0632c Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:49:51 +0300 Subject: [PATCH 09/31] e2e: add CPU list arithmetic helpers. Both test25-irq tests carry byte-identical copies of expand-cpulist and ids-difference, and test19-cpuclass a third copy of expand-cpulist. Two of the copies pass their arguments through expand-cpulist first, so they accept both the compact and the expanded form, while the balloons copies only accept one form each. Keep the permissive behaviour. ids-difference is renamed to cpulist-difference to say what it operates on, since it now lives next to expand-cpulist in the shared library. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 54 +++++++++++++++++++ .../balloons/n4c16/test25-irq/code.var.sh | 30 +---------- .../n4c16/test19-cpuclass/code.var.sh | 24 --------- .../n4c16/test25-irq/code.var.sh | 48 ++--------------- 4 files changed, 59 insertions(+), 97 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 412f4ab29..15f6f2405 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,60 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### CPU lists +### + +expand-cpulist() { # script API + # Usage: expand-cpulist CPULIST + # + # Print the CPUs in CPULIST as a sorted list of space separated CPU ids: + # expand-cpulist "0-2,5" prints "0 1 2 5" + # + # A list which is already in the expanded form is printed as it is, so + # this is safe to use for normalizing a list of unknown form. + local cpus="$1" + + if [ "${cpus//-/}" == "$cpus" ] && [ "${cpus//,/}" == "$cpus" ]; then + echo $cpus + return 0 + fi + + python3 -c ' +import sys +r = set() +for part in sys.argv[1].split(","): + if not part: + continue + if "-" in part: + a, b = part.split("-") + r.update(range(int(a), int(b) + 1)) + else: + r.add(int(part)) +print(" ".join(str(x) for x in sorted(r))) +' "$cpus" +} + +cpulist-difference() { # script API + # Usage: cpulist-difference CPULIST1 CPULIST2 + # + # Print the CPUs which are in CPULIST1 but not in CPULIST2, as a sorted + # list of space separated CPU ids: + # cpulist-difference "1 2 3 4" "2 3" prints "1 4" + # + # Both lists can be given in either the compact or the expanded form. + local ids1="$1" ids2="$2" + + ids1=$(expand-cpulist "$ids1") + ids2=$(expand-cpulist "$ids2") + + python3 -c 'import sys +allc = set(int(x) for x in sys.argv[1].split()) +iso = set(int(x) for x in sys.argv[2].split()) +print(" ".join(str(x) for x in sorted(allc - iso))) +' "$ids1" "$ids2" +} + ### ### Extended resources of the node ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index 522190869..1ff1f6537 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -1,31 +1,5 @@ # Test balloons IRQ CPU affinity: irqClaim and irqMode (sink, isolate). -# expand-cpulist "0-2,5" prints "0 1 2 5" -expand-cpulist() { - python3 -c ' -import sys -r = set() -for part in sys.argv[1].split(","): - if not part: - continue - if "-" in part: - a, b = part.split("-") - r.update(range(int(a), int(b) + 1)) - else: - r.add(int(part)) -print(" ".join(str(x) for x in sorted(r))) -' "$1" -} - -# ids-difference "1 2 3 4" "2 3" prints "1 4" -ids-difference() { - python3 -c 'import sys -allc = set(int(x) for x in sys.argv[1].split()) -iso = set(int(x) for x in sys.argv[2].split()) -print(" ".join(str(x) for x in sorted(allc - iso))) -' "$1" "$2" -} - # ctr-cpu-ids podXcY prints sorted CPU ids allowed for the container, # e.g. "0 1". Requires a preceding "verify" to refresh the state. ctr-cpu-ids() { @@ -109,7 +83,7 @@ echo "isolate CPUs: $isolate_cpus" # iso = set(int(x) for x in sys.argv[2].split()) # print(" ".join(str(x) for x in sorted(allc - iso))) # ' "$ALL_CPUS" "$isolate_cpus") -expected_isolate=$(ids-difference "$ALL_CPUS" "$isolate_cpus") +expected_isolate=$(cpulist-difference "$ALL_CPUS" "$isolate_cpus") verify-irq-cpus "$RTC0_IRQ" "$expected_isolate" @@ -158,7 +132,7 @@ d_claimer_cpus=$(ctr-cpu-ids pod4c0) echo "dedicated claimer CPUs: $d_claimer_cpus" verify-irq-cpus "$RTC0_IRQ" "$d_claimer_cpus" -expected_isolate=$(ids-difference "$ALL_CPUS" "$d_claimer_cpus") +expected_isolate=$(cpulist-difference "$ALL_CPUS" "$d_claimer_cpus") verify-irq-cpus "$TTYS0_IRQ" "$expected_isolate" verify-irq-cpus "$ACPI_IRQ" "$expected_isolate" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index c8be809de..53b89e13f 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -38,30 +38,6 @@ assert-cpu-freq() { done } -# expand-cpulist "0-2,5" prints "0 1 2 5" -expand-cpulist() { - local cpus="$1" - - if [ "${cpus//-/}" == "$cpus" ] && [ "${cpus//,/}" == "$cpus" ]; then - echo $cpus - return 0 - fi - - python3 -c ' -import sys -r = set() -for part in sys.argv[1].split(","): - if not part: - continue - if "-" in part: - a, b = part.split("-") - r.update(range(int(a), int(b) + 1)) - else: - r.add(int(part)) -print(" ".join(str(x) for x in sorted(r))) -' "$cpus" -} - OVERRIDE_SYS_CPUFREQ='[{"cpus": "0-15", "base": 2900000, "min": 800000, "max": 3800000}]' OVERRIDE_SST='{"supported": true, "clos_count": 4, "packages": [{"id": 0, "cpus": "0-7", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}, {"id": 1, "cpus": "8-15", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}]}' OVERRIDE_SST_STATE_DIR="/tmp/nri-pct-mock" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh index 323d89b1c..ef573dd33 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh @@ -61,48 +61,6 @@ resolve-irq() { fi } -# expand-cpulist "0-2,5" prints "0 1 2 5" -expand-cpulist() { - local cpus="$1" - - if [ "${cpus//-/}" == "$cpus" ] && [ "${cpus//,/}" == "$cpus" ]; then - echo $cpus - return 0 - fi - - python3 -c ' -import sys -r = set() -for part in sys.argv[1].split(","): - if not part: - continue - if "-" in part: - a, b = part.split("-") - r.update(range(int(a), int(b) + 1)) - else: - r.add(int(part)) -print(" ".join(str(x) for x in sorted(r))) -' "$cpus" -} - -# ids-difference "1 2 3 4" "2 3" prints "1 4" -ids-difference() { - local ids1="$1" ids2="$2" - - if [ "${ids1//-/}" != "$ids1" ] || [ "${ids1//,/}" != "$ids1" ]; then - ids1=$(expand-cpulist $ids1) - fi - if [ "${ids2//-/}" != "$ids2" ] || [ "${ids2//,/}" != "$ids2" ]; then - ids2=$(expand-cpulist $ids2) - fi - - python3 -c 'import sys -allc = set(int(x) for x in sys.argv[1].split()) -iso = set(int(x) for x in sys.argv[2].split()) -print(" ".join(str(x) for x in sorted(allc - iso))) -' "$ids1" "$ids2" -} - # verify-irq-cpus IRQNUM EXPECTED waits until the affinity of the IRQ # equals EXPECTED (sorted space-separated CPU ids), or fails after a # timeout. @@ -180,8 +138,8 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ CPU=2 CONTCOUNT=4 create guaranteed -verify-irq-cpus ".*ttyS0.*" "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" -verify-irq-cpus ".*rtc0.*" "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c1))" +verify-irq-cpus ".*ttyS0.*" "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" +verify-irq-cpus ".*rtc0.*" "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). delete-pods $pod @@ -212,7 +170,7 @@ ANN0=$ANN0 ANN1=$ANN1 \ CPU=2 CONTCOUNT=4 create guaranteed verify-irq-cpus ".*ttyS0.*" $(ctr-cpu-ids $pod ${pod}c0) -verify-irq-cpus ".*rtc0.*" "$(ids-difference "$(ids-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" $(ctr-cpu-ids $pod ${pod}c1))" +verify-irq-cpus ".*rtc0.*" "$(cpulist-difference "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" $(ctr-cpu-ids $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). delete-pods $pod From 442a4c2d6798f014f36b8f7609e7748c5d5990af Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 15:57:08 +0300 Subject: [PATCH 10/31] e2e: add container cpuset lookup helpers. Four tests look up which CPUs a container is allowed to run on, using two fundamentally different mechanisms: two of them read the live cpuset from inside the container with kubectl exec, and two read it from the snapshot which "report allowed" took. The two are not interchangeable, so keep both under names which say which one they are, and convert each test to the mechanism it already used. container-cpus also gets working error handling. Both kubectl exec copies tested $? after a pipeline, so they checked the exit status of cut rather than that of ssh and kubectl, and never reported anything. allowed-cpu-ids returns the ids sorted, which the copy in test17-cstates-scheduling did not. The test uses them as a set, so this only makes its output deterministic. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 30 +++++++++++++++++++ .../test17-cstates-scheduling/code.var.sh | 19 ++++-------- .../balloons/n4c16/test25-irq/code.var.sh | 18 ++++------- .../n4c16/test19-cpuclass/code.var.sh | 26 +++++----------- .../n4c16/test25-irq/code.var.sh | 29 +++++------------- 5 files changed, 57 insertions(+), 65 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 15f6f2405..4086b7467 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -212,6 +212,36 @@ print(" ".join(str(x) for x in sorted(allc - iso))) ' "$ids1" "$ids2" } +container-cpus() { # script API + # Usage: container-cpus POD CONTAINER + # + # Print the cpuset CONTAINER of POD is currently allowed to run on, as + # read from inside the container. The list is in the compact form, for + # instance "4-5,12". + # + # This is the live cpuset. Use allowed-cpu-ids instead to read the cpuset + # from the latest snapshot which "report allowed" took. + local pod=$1 ctr=$2 status + + status=$(vm-command-q \ + "kubectl exec $pod -c $ctr -- grep Cpus_allowed_list /proc/1/status") || + error "failed to read the cpuset of container $ctr of pod $pod" + + tr -d '\t ' <<< "$status" | cut -d ':' -f 2 +} + +allowed-cpu-ids() { # script API + # Usage: allowed-cpu-ids CONTAINER + # + # Print the CPU ids CONTAINER is allowed to run on, as a sorted list of + # space separated ids, for instance "4 5 12". + # + # This reads the latest snapshot which "report allowed" took, so it needs + # a preceding report or verify. Use container-cpus instead to read the + # live cpuset from inside the container. + pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" +} + ### ### Extended resources of the node ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index 47b541424..781c1c418 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -3,13 +3,6 @@ helm-terminate helm_config=$TEST_DIR/balloons-cstates.cfg helm-launch balloons -# cpuids-of-container returns CPU ids a container is allowed to use, e.g. "1 2 4" -cpuids-of() { - local ctr=$1 # e.g. pod0c0 - # return only cpu ids without zero-fill: replace cpu01 -> 1, cpu11 -> 11 - pyexec "for cpu in cpus['$ctr']: print(cpu.replace('cpu0','').replace('cpu',''))" -} - # verify-cstates checks the last writes to "disable" files in the # override fs. verify-cstates() { @@ -80,7 +73,7 @@ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOU report allowed verify 'len(cpus["pod0c0"]) == 1' echo "verify that CPUs of low-latency pod0 cannot enter C4 or C8" -verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 4 +verify-cstates "$(allowed-cpu-ids pod0c0)" "C1E C2" "C4 C8" 4 expected_policy=1 expected_prio=$((99 - 42)) verify-sched pod0c0 # expect SCHED_FIFO, prio 56 CPUREQ="3" MEMREQ="100M" CPULIM="" MEMLIM="" @@ -93,7 +86,7 @@ report allowed verify 'cpus["pod0c0"] == cpus["pod1c0"]' \ 'len(cpus["pod0c0"]) == 4' echo "verify that CPUs of low-latency pods pod0 and pod1 cannot enter C4 or C8" -verify-cstates "$(cpuids-of pod1c0)" "C1E C2" "C4 C8" 16 +verify-cstates "$(allowed-cpu-ids pod1c0)" "C1E C2" "C4 C8" 16 expected_policy=5 expected_prio=$((120 + 17)) verify-sched pod1c0 # expect SCHED_IDLE, prio 137 vm-command "ionice -p \$(pgrep -f 'echo pod1c0')" || @@ -103,14 +96,14 @@ expected_ionice="best-effort: prio 6" command-error "expected ionice output '$expected_ionice'" # store CPU ids of maximal cpuset before deleting pods -max_lowlatency_cpus="$(echo $(cpuids-of pod1c0) )" +max_lowlatency_cpus="$(echo $(allowed-cpu-ids pod1c0) )" vm-command 'kubectl delete pod pod1' report allowed verify 'len(cpus["pod0c0"]) == 1' # spaces around each id helps ensuring grep " 1 " never matches cpu 11 but always matches cpu 1 -pod0cpus=" $(echo $(cpuids-of pod0c0) ) " +pod0cpus=" $(echo $(allowed-cpu-ids pod0c0) ) " echo "verify that c-states of freed CPUs are enabled again after balloon was deflated" freed_cpus="" @@ -120,8 +113,8 @@ done echo "verify that all c-states of freed CPUs $freed_cpus (= {$max_lowlatency_cpus} - {$pod0cpus}) are enabled after the balloon got deflated" verify-cstates "$freed_cpus" "C1E C2 C4 C8" "" 24 -echo "verify that c-states of the remaining CPU $(cpuids-of pod0c0) are still configured for low-latency" -verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 16 +echo "verify that c-states of the remaining CPU $(allowed-cpu-ids pod0c0) are still configured for low-latency" +verify-cstates "$(allowed-cpu-ids pod0c0)" "C1E C2" "C4 C8" 16 vm-command 'kubectl delete pod pod0' report allowed diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index 1ff1f6537..528a55cf9 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -1,11 +1,5 @@ # Test balloons IRQ CPU affinity: irqClaim and irqMode (sink, isolate). -# ctr-cpu-ids podXcY prints sorted CPU ids allowed for the container, -# e.g. "0 1". Requires a preceding "verify" to refresh the state. -ctr-cpu-ids() { - pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" -} - # irq-cpu-ids IRQNUM prints sorted CPU ids in the affinity of the IRQ. irq-cpu-ids() { expand-cpulist "$(vm-command-q "cat /proc/irq/$1/smp_affinity_list" | tr -d '[:space:]')" @@ -59,7 +53,7 @@ helm_config=${TEST_DIR}/balloons-irq-claim.cfg helm-launch balloons POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox report allowed -claimer_cpus=$(ctr-cpu-ids pod0c0) +claimer_cpus=$(allowed-cpu-ids pod0c0) echo "claimer CPUs: $claimer_cpus" verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" @@ -75,7 +69,7 @@ cleanup POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: isolate" CONTCOUNT=1 create balloons-busybox report allowed -isolate_cpus=$(ctr-cpu-ids pod1c0) +isolate_cpus=$(allowed-cpu-ids pod1c0) echo "isolate CPUs: $isolate_cpus" # expected_isolate=$(python3 -c 'import sys @@ -98,7 +92,7 @@ verify-irq-cpus "$TTYS0_IRQ" "$ALL_CPUS" POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: sink" CONTCOUNT=1 create balloons-busybox report allowed verify -sink_cpus=$(ctr-cpu-ids pod2c0) +sink_cpus=$(allowed-cpu-ids pod2c0) echo "sink CPUs: $sink_cpus" verify-irq-cpus "$TTYS0_IRQ" "$sink_cpus" verify-irq-cpus "$RTC0_IRQ" "$sink_cpus" @@ -106,8 +100,8 @@ verify-irq-cpus "$ACPI_IRQ" "$sink_cpus" POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox report allowed -sink_cpus=$(ctr-cpu-ids pod2c0) -claimer_cpus=$(ctr-cpu-ids pod3c0) +sink_cpus=$(allowed-cpu-ids pod2c0) +claimer_cpus=$(allowed-cpu-ids pod3c0) echo "claimer CPUs: $claimer_cpus" echo "sink CPUs: $sink_cpus" verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" @@ -128,7 +122,7 @@ cleanup POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: dedicated-claimer" CONTCOUNT=1 create balloons-busybox report allowed -d_claimer_cpus=$(ctr-cpu-ids pod4c0) +d_claimer_cpus=$(allowed-cpu-ids pod4c0) echo "dedicated claimer CPUs: $d_claimer_cpus" verify-irq-cpus "$RTC0_IRQ" "$d_claimer_cpus" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 53b89e13f..6f3ec416b 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -6,16 +6,6 @@ cleanup() { # Restrict the log assertions to the cpuclass-related log lines. plugin_log_filter=' *cpuclass *' -# ctr-cpu-ids -# Inspect the container of the given pod and report the cpuset it is pinned to. -ctr-cpu-ids() { - local pod=$1 ctr=$2 - local result="" - $SSH -oConnectTimeout=1 node \ - "kubectl exec $pod -c $ctr -- grep Cpus_allowed_list /proc/1/status" | \ - tr -d '\t ' | cut -d ':' -f 2 -} - # assert-cpu-clos # Polls the log until a default timeout to verify that the given CPUs are # associated to the given CLOS. @@ -91,13 +81,13 @@ assert-cpu-freq "reserved pool" $cpu0 reserved CONTCOUNT=4 CPU=1 create guaranteed pod=pod0 -cpu0=$(ctr-cpu-ids $pod ${pod}c0) +cpu0=$(container-cpus $pod ${pod}c0) assert-cpu-clos ${pod}c0 $cpu0 "CLOS 0" -cpu1=$(ctr-cpu-ids $pod ${pod}c1) +cpu1=$(container-cpus $pod ${pod}c1) assert-cpu-clos ${pod}c1 $cpu1 "CLOS 0" -cpu2=$(ctr-cpu-ids $pod ${pod}c2) +cpu2=$(container-cpus $pod ${pod}c2) assert-cpu-clos ${pod}c2 $cpu2 "CLOS 0" -cpu3=$(ctr-cpu-ids $pod ${pod}c3) +cpu3=$(container-cpus $pod ${pod}c3) assert-cpu-clos ${pod}c3 $cpu3 "CLOS 0" # Delete pod. Verify that each released exclusive CPU gets assigned to @@ -121,13 +111,13 @@ ANN0="cpu-class.resource-policy.nri.io/container.${pod}c0: class1" \ ANN1="cpu-class.resource-policy.nri.io/container.${pod}c1: class2" \ CONTCOUNT=4 CPU=2 create guaranteed -cpu0=$(ctr-cpu-ids $pod ${pod}c0) +cpu0=$(container-cpus $pod ${pod}c0) assert-cpu-freq ${pod}c0 $cpu0 class1 -cpu1=$(ctr-cpu-ids $pod ${pod}c1) +cpu1=$(container-cpus $pod ${pod}c1) assert-cpu-freq ${pod}c1 $cpu1 class2 -cpu2=$(ctr-cpu-ids $pod ${pod}c2) +cpu2=$(container-cpus $pod ${pod}c2) assert-cpu-clos ${pod}c2 $cpu2 "CLOS 0" -cpu3=$(ctr-cpu-ids $pod ${pod}c3) +cpu3=$(container-cpus $pod ${pod}c3) assert-cpu-clos ${pod}c3 $cpu3 "CLOS 0" # Delete pod. Verify that each released CPU get assigned to diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh index ef573dd33..11f87f581 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh @@ -3,21 +3,6 @@ cleanup() { helm-terminate } -# ctr-cpu-ids -# Inspect the container of the given pod and report the cpuset it is pinned to. -ctr-cpu-ids() { - local pod=$1 ctr=$2 - local cpus="" - - $SSH -oConnectTimeout=1 node \ - "kubectl exec $pod -c $ctr -- grep Cpus_allowed_list /proc/1/status" | \ - tr -d '\t ' | cut -d ':' -f 2 - if [ $? -ne 0 ]; then - error "Failed to get cpuset for container $ctr in pod $pod" >&2 - return 1 - fi -} - # irq-cpu-ids # Read the current affinity for the given interrupt from /proc/irq/$irq/smp_affinity_list. irq-cpu-ids() { @@ -109,8 +94,8 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ CPU=2 CONTCOUNT=4 create guaranteed -verify-irq-cpus ".*ttyS0.*" "$(ctr-cpu-ids $pod ${pod}c0)" -verify-irq-cpus ".*rtc0.*" "$(ctr-cpu-ids $pod ${pod}c1)" +verify-irq-cpus ".*ttyS0.*" "$(container-cpus $pod ${pod}c0)" +verify-irq-cpus ".*rtc0.*" "$(container-cpus $pod ${pod}c1)" # Delete pod and check that the IRQ affinities are restored to all CPUs. delete-pods $pod @@ -138,8 +123,8 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ CPU=2 CONTCOUNT=4 create guaranteed -verify-irq-cpus ".*ttyS0.*" "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" -verify-irq-cpus ".*rtc0.*" "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c1))" +verify-irq-cpus ".*ttyS0.*" "$(cpulist-difference $ALLCPUS $(container-cpus $pod ${pod}c0))" +verify-irq-cpus ".*rtc0.*" "$(cpulist-difference $ALLCPUS $(container-cpus $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). delete-pods $pod @@ -169,8 +154,8 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ CPU=2 CONTCOUNT=4 create guaranteed -verify-irq-cpus ".*ttyS0.*" $(ctr-cpu-ids $pod ${pod}c0) -verify-irq-cpus ".*rtc0.*" "$(cpulist-difference "$(cpulist-difference $ALLCPUS $(ctr-cpu-ids $pod ${pod}c0))" $(ctr-cpu-ids $pod ${pod}c1))" +verify-irq-cpus ".*ttyS0.*" $(container-cpus $pod ${pod}c0) +verify-irq-cpus ".*rtc0.*" "$(cpulist-difference "$(cpulist-difference $ALLCPUS $(container-cpus $pod ${pod}c0))" $(container-cpus $pod ${pod}c1))" # Delete pod and check that the IRQ affinities are restored to the default (all CPUs). delete-pods $pod @@ -314,7 +299,7 @@ EOF ANN0=$ANN0 ANN1=$ANN1 \ CONTCOUNT=1 create guaranteed -verify-irq-cpus ".*ttyS0.*" "$(ctr-cpu-ids $pod ${pod}c0)" +verify-irq-cpus ".*ttyS0.*" "$(container-cpus $pod ${pod}c0)" delete-pods $pod unset ANN0 ANN1 From 90f9bbd0d8908adb79fc33acc2bfb8cd4ccbe547 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:03:34 +0300 Subject: [PATCH 11/31] e2e: add interrupt affinity test helpers. The two test25-irq tests are the pair with the most duplication in the suite. Together with the CPU list and cpuset helpers already lifted out, this shrinks them from 551 to 307 lines. The two copies of verify-irq-cpus differ in that the balloons one takes an interrupt number, which the test resolves from /proc/interrupts with awk of its own, while the topology-aware one takes a pattern and resolves it itself. Keep resolve-irq, which accepts both, and let the balloons test name its interrupts by pattern too. resolve-irq now also rejects a match which is not an interrupt number. A pattern matching no interrupt used to fall through to matching the header line of /proc/interrupts, and resolved to the concatenation of its CPU column titles instead of reporting that there is no such interrupt. set-irq-cpus is not lifted out. It had no callers. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 60 ++++++++++++++++++ .../balloons/n4c16/test25-irq/code.var.sh | 39 ++---------- .../n4c16/test25-irq/code.var.sh | 61 ------------------- 3 files changed, 64 insertions(+), 96 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 4086b7467..b0d6a4e2d 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -242,6 +242,66 @@ allowed-cpu-ids() { # script API pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" } +### +### Interrupts +### + +resolve-irq() { # script API + # Usage: resolve-irq IRQ + # + # Print the number of interrupt IRQ, which is either an interrupt number + # or an extended regular expression matching a line in /proc/interrupts, + # for instance ".*ttyS0.*". Fail the test if there is no such interrupt. + local irq_or_pattern="$1" irq="" interrupts + + interrupts=$(vm-command-q "cat /proc/interrupts" | tr -s ' \t' ' ') + + # Try an interrupt number first, then a pattern. + irq=$(grep "^ *$irq_or_pattern:" <<< "$interrupts" | cut -d ':' -f1 | tr -d ' ' | head -n 1) + if [ -z "$irq" ]; then + irq=$(grep -E "$irq_or_pattern" <<< "$interrupts" | cut -d ':' -f1 | tr -d ' ' | head -n 1) + fi + # Reject a match which is not an interrupt number. Without this, a pattern + # which happens to match the header line, or one of the non-numbered + # counters at the end of the file, would resolve to garbage. + if ! [[ "$irq" =~ ^[0-9]+$ ]]; then + error "no interrupt matching \"$irq_or_pattern\" found in /proc/interrupts" + fi + if [ "$irq" != "$irq_or_pattern" ]; then + echo "interrupt \"$irq_or_pattern\" resolved to irq $irq" >&2 + fi + echo "$irq" +} + +irq-cpu-ids() { # script API + # Usage: irq-cpu-ids IRQ + # + # Print the CPU ids in the affinity of interrupt IRQ, as a sorted list of + # space separated ids. IRQ is resolved with resolve-irq. + local irq + irq=$(resolve-irq "$1") || return 1 + expand-cpulist "$(vm-command-q "cat /proc/irq/$irq/smp_affinity_list" | tr -d '[:space:]')" +} + +verify-irq-cpus() { # script API + # Usage: verify-irq-cpus IRQ EXPECTED [TIMEOUT] + # + # Wait until the affinity of interrupt IRQ becomes EXPECTED, TIMEOUT + # seconds at most, 20 by default. Fail the test otherwise. + # + # Both the expected and the observed CPU list are normalized, so EXPECTED + # can be given in either the compact or the expanded form. + local irq expected got tmo=${3:-20} + + irq=$(resolve-irq "$1") + expected=$(expand-cpulist "$2") + + retry-until --timeout "$tmo" 'got=$(irq-cpu-ids "$irq"); [ "$got" == "$expected" ]' || { + error "irq $irq affinity: expected CPUs '$expected', got '$got'" + } + echo "irq $irq affinity is '$got' as expected" +} + ### ### Extended resources of the node ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index 528a55cf9..b0307c706 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -1,40 +1,9 @@ # Test balloons IRQ CPU affinity: irqClaim and irqMode (sink, isolate). -# irq-cpu-ids IRQNUM prints sorted CPU ids in the affinity of the IRQ. -irq-cpu-ids() { - expand-cpulist "$(vm-command-q "cat /proc/irq/$1/smp_affinity_list" | tr -d '[:space:]')" -} - -# set-irq-cpus IRQNUM CPULIST sets the affinity of the IRQ, e.g. "0-15". -set-irq-cpus() { - vm-command "echo $2 > /proc/irq/$1/smp_affinity_list" || - command-error "failed to set affinity of irq $1" -} - -# verify-irq-cpus IRQNUM EXPECTED waits until the affinity of the IRQ -# equals EXPECTED (sorted space-separated CPU ids), or fails after a -# timeout. -verify-irq-cpus() { - local irqnum=$1 expected=$2 got tries=20 - while [ "$tries" -gt 0 ]; do - got=$(irq-cpu-ids "$irqnum") - if [ "$got" == "$expected" ]; then - echo "irq $irqnum affinity is '$got' as expected" - return 0 - fi - tries=$((tries - 1)) - sleep 1 - done - error "irq $irqnum affinity: expected CPUs '$expected', got '$got'" -} - -# Detect ttyS0 and rtc0 is IRQ numbers on vm. -vm-command "awk '/ ttyS0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" -TTYS0_IRQ=$COMMAND_OUTPUT -vm-command "awk '/ rtc0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" -RTC0_IRQ=$COMMAND_OUTPUT -vm-command "awk '/ acpi/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" -ACPI_IRQ=$COMMAND_OUTPUT +# Interrupts used in this test, resolved from /proc/interrupts on demand. +TTYS0_IRQ=".* ttyS0.*" +RTC0_IRQ=".* rtc0.*" +ACPI_IRQ=".* acpi.*" ALL_CPUS="$(expand-cpulist 0-15)" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh index 11f87f581..6547268bb 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test25-irq/code.var.sh @@ -3,67 +3,6 @@ cleanup() { helm-terminate } -# irq-cpu-ids -# Read the current affinity for the given interrupt from /proc/irq/$irq/smp_affinity_list. -irq-cpu-ids() { - local pattern=$1 - local irq="" cpus="" - - irq=$(resolve-irq "$pattern") - if [ -z "$irq" ]; then - error "Failed to resolve IRQ for pattern: $pattern" >&2 - return 1 - fi - $SSH -oConnectTimeout=1 node "cat /proc/irq/$irq/smp_affinity_list" - if [ $? -ne 0 ]; then - error "Failed to read smp_affinity_list for IRQ $irq" >&2 - return 1 - fi -} - -# resolve-irq -# Returns the IRQ number matching the given interrupt or pattern in /proc/interrupts. -resolve-irq() { - local irq_or_pattern="$1" - local irq="" - - irq=$($SSH -oConnectTimeout=1 node "cat /proc/interrupts" | \ - tr -s ' \t' ' ' | grep "^ *$irq_or_pattern:" | cut -d ':' -f1 | tr -d ' ') - if [ -z "$irq" ]; then - irq=$($SSH -oConnectTimeout=1 node "cat /proc/interrupts" | \ - tr -s ' \t' ' ' | grep -E "$irq_or_pattern" | cut -d ':' -f1 | tr -d ' ') - fi - - if [ -n "$irq" ]; then - if [ "$irq" != "$irq_or_pattern" ]; then - echo "IRQ $irq_or_pattern resolved to $irq..." >&2 - fi - echo $irq - return 0 - else - echo "IRQ not found for pattern: $irq_or_pattern" >&2 - return 1 - fi -} - -# verify-irq-cpus IRQNUM EXPECTED waits until the affinity of the IRQ -# equals EXPECTED (sorted space-separated CPU ids), or fails after a -# timeout. -verify-irq-cpus() { - local irq=$1 expected=$2 got tries=20 - irqnum=$(resolve-irq "$irq") - while [ "$tries" -gt 0 ]; do - got=$(irq-cpu-ids "$irqnum") - if [ "$(expand-cpulist "$got")" == "$(expand-cpulist "$expected")" ]; then - echo "IRQ $irqnum affinity is '$got' as expected" - return 0 - fi - tries=$((tries - 1)) - sleep 1 - done - error "IRQ $irqnum affinity: expected CPUs '$expected', got '$got'" -} - cleanup DEBUG_LOGGERS="irq" From 746e54369abc85d492a7ea8041d1b5e10e0a1679 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:04:32 +0300 Subject: [PATCH 12/31] e2e: add scheduling class verification helper. Both test17 tests define verify-sched. The topology-aware copy is the stricter one: it fails if the expected policy or priority is not given, instead of silently verifying nothing, and it drops the uninteresting lines of /proc/PID/sched from the output. Keep that one. The balloons test spelled the expected policies as bare numbers with the name in a comment. It now uses the same SCHED_* constants which the topology-aware test defined for itself. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 42 +++++++++++++++++++ .../test17-cstates-scheduling/code.var.sh | 22 +--------- .../test17-scheduling-classes/code.var.sh | 28 ------------- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index b0d6a4e2d..832617282 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -242,6 +242,48 @@ allowed-cpu-ids() { # script API pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" } +### +### Scheduling +### + +# Scheduling policies, as reported in /proc/PID/sched. +SCHED_OTHER=0 +SCHED_FIFO=1 +SCHED_RR=2 +SCHED_BATCH=3 +SCHED_ISO=4 +SCHED_IDLE=5 +SCHED_DEADLINE=6 + +verify-sched() { # script API + # Usage: expected_policy=POLICY expected_prio=PRIO verify-sched CONTAINER + # + # Verify the scheduling policy and priority of the process of CONTAINER. + # POLICY is one of the SCHED_* constants above. Fail the test unless both + # expected values are given, so that a typo in a variable name cannot turn + # the verification into a no-op. + local podXcY=$1 + + vm-command "cat /proc/\$(pgrep -f 'echo $podXcY')/sched | grep -E '^((policy)|(prio))'" || + command-error "cannot get /proc/PID/sched for $podXcY" + + if [ "$expected_policy" != "" ]; then + echo "verify scheduling policy of $podXcY is $expected_policy" + grep -q -E "policy .* $expected_policy" <<< $COMMAND_OUTPUT || + error "expected policy $expected_policy not found" + else + error "missing verify-sched expected_policy for $podXcY" + fi + + if [ "$expected_prio" != "" ]; then + echo "verify scheduling priority of $podXcY is $expected_prio" + grep -q -E "prio .* $expected_prio" <<< $COMMAND_OUTPUT || + error "expected priority $expected_prio not found" + else + error "missing verify-sched expected_prio for $podXcY" + fi +} + ### ### Interrupts ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index 781c1c418..4ab6ae759 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -28,24 +28,6 @@ verify-cstates() { done } -verify-sched() { - local podXcY=$1 - vm-command "cat /proc/\$(pgrep -f 'echo $podXcY')/sched" || command-error "cannot get /proc/PID/sched for $podXcY" - - if [ "$expected_policy" != "" ]; then - echo "verify scheduling policy of $podXcY is $expected_policy" - grep -q -E "policy .* $expected_policy" <<< $COMMAND_OUTPUT || - error "expected policy $expected_policy not found" - - fi - - if [ "$expected_prio" != "" ]; then - echo "verify scheduling priority of $podXcY is $expected_prio" - grep -q -E "prio .* $expected_prio" <<< $COMMAND_OUTPUT || - error "expected priority $expected_prio not found" - fi -} - # verify-cstates-no-writes checks that any c-states of given CPUs have not been written verify-cstates-no-writes() { local cpu_ids=$1 # e.g. "1 2 4" @@ -74,7 +56,7 @@ report allowed verify 'len(cpus["pod0c0"]) == 1' echo "verify that CPUs of low-latency pod0 cannot enter C4 or C8" verify-cstates "$(allowed-cpu-ids pod0c0)" "C1E C2" "C4 C8" 4 -expected_policy=1 expected_prio=$((99 - 42)) verify-sched pod0c0 # expect SCHED_FIFO, prio 56 +expected_policy=$SCHED_FIFO expected_prio=$((99 - 42)) verify-sched pod0c0 # prio 56 CPUREQ="3" MEMREQ="100M" CPULIM="" MEMLIM="" POD_ANNOTATION=( @@ -88,7 +70,7 @@ verify 'cpus["pod0c0"] == cpus["pod1c0"]' \ echo "verify that CPUs of low-latency pods pod0 and pod1 cannot enter C4 or C8" verify-cstates "$(allowed-cpu-ids pod1c0)" "C1E C2" "C4 C8" 16 -expected_policy=5 expected_prio=$((120 + 17)) verify-sched pod1c0 # expect SCHED_IDLE, prio 137 +expected_policy=$SCHED_IDLE expected_prio=$((120 + 17)) verify-sched pod1c0 # prio 137 vm-command "ionice -p \$(pgrep -f 'echo pod1c0')" || command-error "cannot get ionice for pod1c0" expected_ionice="best-effort: prio 6" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh index cd1cba6b3..494c03dd7 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test17-scheduling-classes/code.var.sh @@ -3,34 +3,6 @@ cleanup() { delete-namespaces highprio lowprio } -verify-sched() { - local podXcY=$1 - vm-command "cat /proc/\$(pgrep -f 'echo $podXcY')/sched | grep -E '^((policy)|(prio))'" || command-error "cannot get /proc/PID/sched for $podXcY" - - if [ "$expected_policy" != "" ]; then - echo "verify scheduling policy of $podXcY is $expected_policy" - grep -q -E "policy .* $expected_policy" <<< $COMMAND_OUTPUT || - error "expected policy $expected_policy not found" - else - error "missing verify-sched expected_policy for $podXcY" - fi - - if [ "$expected_prio" != "" ]; then - echo "verify scheduling priority of $podXcY is $expected_prio" - grep -q -E "prio .* $expected_prio" <<< $COMMAND_OUTPUT || - error "expected priority $expected_prio not found" - else - error "missing verify-sched expected_prio for $podXcY" - fi -} - -SCHED_OTHER=0 -SCHED_FIFO=1 -SCHED_BATCH=3 -SCHED_ISO=4 -SCHED_IDLE=5 -SCHED_DEADLINE=6 - SCHEDULING_CLASSES="[ { name: realtime, policy: fifo, priority: 42 }, { name: highprio, policy: fifo, priority: 10 }, From 7ffa8ccffa0f0ec89ddd8f04a0daece3d409eded Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:05:58 +0300 Subject: [PATCH 13/31] e2e: add policy configuration status helpers. test20-config-status and test12-config-status are the same test for two different policies, and both spell out the jsonpath of the node status, the kubectl wait, and the status dump for the failure case. Three more blocks in test19-cpuclass check the errors in the status the same way. The helpers go to run.sh next to get-config-node-status-result and its relatives, which already know how to address the configuration resource of a policy, rather than into lib/test.bash where they would start a second, parallel API for the same thing. They also address the node like the rest of that family does, with get-hostname-for-vm, instead of using $VM_HOSTNAME directly. The two differ if the hostname of the VM has a domain part. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../n4c16/test20-config-status/code.var.sh | 17 +------ .../n4c16/test12-config-status/code.var.sh | 17 +------ .../n4c16/test19-cpuclass/code.var.sh | 15 ++---- test/e2e/run.sh | 49 +++++++++++++++++++ 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh index 2b27ab423..37ef2c35d 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh @@ -3,26 +3,13 @@ helm_config=$TEST_DIR/balloons.cfg helm-launch balloons sleep 1 -jsonpath="{.status.nodes['$VM_HOSTNAME'].status}" -vm-command "kubectl wait -n kube-system balloonspolicies/default \ - --for=jsonpath=\"$jsonpath\"=\"Success\" --timeout=5s" || { - echo "Unexpected config status:" - vm-command "kubectl get -n kube-system balloonspolicies/default \ - -o jsonpath=\"{.status}\" | jq ." - error "expected initial Success status" -} +wait-config-status Success host-command "$SCP $TEST_DIR/broken-balloons-config.yaml ${VM_HOSTNAME}:" vm-command "kubectl apply -f broken-balloons-config.yaml" sleep 1 -vm-command "kubectl wait -n kube-system balloonspolicies/default \ - --for=jsonpath=\"$jsonpath\"=\"Failure\" --timeout=5s" || { - echo "Unexpected config status:" - vm-command "kubectl get -n kube-system balloonspolicies/default \ - -o jsonpath=\"{.status}\" | jq ." - error "expected post-update Failure status" -} +wait-config-status Failure helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test12-config-status/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test12-config-status/code.var.sh index c8472e046..f1f829fae 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test12-config-status/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test12-config-status/code.var.sh @@ -12,14 +12,7 @@ helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware sleep 1 -jsonpath="{.status.nodes['$VM_HOSTNAME'].status}" -vm-command "kubectl wait -n kube-system topologyawarepolicies/default \ - --for=jsonpath=\"$jsonpath\"=\"Success\" --timeout=5s" || { - echo "Unexpected config status:" - vm-command "kubectl get -n kube-system topologyawarepolicies/default \ - -o jsonpath=\"{.status}\" | jq ." - error "expected initial Success status" -} +wait-config-status Success # verify propagation of errors back to source CR vm-put-file $(RESERVED_CPU=750x instantiate custom-config.yaml) broken-config.yaml @@ -27,13 +20,7 @@ vm-command "kubectl apply -f broken-config.yaml" sleep 1 -vm-command "kubectl wait -n kube-system topologyawarepolicies/default \ - --for=jsonpath=\"$jsonpath\"=\"Failure\" --timeout=5s" || { - echo "Unexpected config status:" - vm-command "kubectl get -n kube-system topologyawarepolicies/default \ - -o jsonpath=\"{.status}\" | jq ." - error "expected post-update Failure status" -} +wait-config-status Failure helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 6f3ec416b..31d33f2db 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -175,10 +175,7 @@ helm_config=$(COLOCATE_PODS=false \ EXTRA_ENV_OVERRIDE_SST_STATE_DIR="$OVERRIDE_SST_STATE_DIR" \ instantiate helm-config.yaml) launch_timeout=5s expect_error=1 helm-launch topology-aware -vm-command "kubectl -n kube-system get topologyawarepolicies.config.nri/default -ojson | jq '.status.nodes[].errors'" - -grep -q 'unknown reserved CPU class \\"nonexistent\\"' <<<$COMMAND_OUTPUT || - error "Missing reserved pool CPU class validation error in configuration CR" +verify-config-status-error 'unknown reserved CPU class \\"nonexistent\\"' RESERVED_CPUCLASS=reserved helm-terminate @@ -197,10 +194,7 @@ helm_config=$(COLOCATE_PODS=false \ EXTRA_ENV_OVERRIDE_SST_STATE_DIR="$OVERRIDE_SST_STATE_DIR" \ instantiate helm-config.yaml) launch_timeout=5s expect_error=1 helm-launch topology-aware -vm-command "kubectl -n kube-system get topologyawarepolicies.config.nri/default -ojson | jq '.status.nodes[].errors'" - -grep -q 'unknown shared CPU class \\"nonexistent\\"' <<<$COMMAND_OUTPUT || - error "Missing shared pool CPU class validation error in configuration CR" +verify-config-status-error 'unknown shared CPU class \\"nonexistent\\"' SHARED_CPUCLASS=shared helm-terminate @@ -219,9 +213,6 @@ helm_config=$(COLOCATE_PODS=false \ EXTRA_ENV_OVERRIDE_SST_STATE_DIR="$OVERRIDE_SST_STATE_DIR" \ instantiate helm-config.yaml) launch_timeout=5s expect_error=1 helm-launch topology-aware -vm-command "kubectl -n kube-system get topologyawarepolicies.config.nri/default -ojson | jq '.status.nodes[].errors'" - -grep -q 'unknown default exclusive CPU class \\"nonexistent\\"' <<<$COMMAND_OUTPUT || - error "Missing default exclusive CPU class validation error in configuration CR" +verify-config-status-error 'unknown default exclusive CPU class \\"nonexistent\\"' cleanup diff --git a/test/e2e/run.sh b/test/e2e/run.sh index ff6fe9df8..aa4c7b81d 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -631,6 +631,55 @@ wait-config-node-status() { error "waiting for node $node update in $resource failed" } +config-resource() { # script API + # Usage: config-resource [POLICY] + # + # Print the name of the configuration custom resource of POLICY, $POLICY + # by default. + # + # This mirrors the cfgresource defaults of helm-launch. + local policy=${1:-$POLICY} + case "$policy" in + *topology*aware*) echo topologyawarepolicies/default;; + *balloons*) echo balloonspolicies/default;; + *) error "config-resource: unknown policy \"$policy\"";; + esac +} + +wait-config-status() { # script API + # Usage: wait-config-status STATUS [TIMEOUT] [RESOURCE] + # + # Wait until the node status of the configuration custom resource RESOURCE + # becomes STATUS, that is, Success or Failure. RESOURCE defaults to the + # configuration resource of $POLICY and TIMEOUT to 5s. + # + # Dump the whole status of the resource and fail the test on timeout. + local status="$1" timeout="${2:-5s}" resource="${3:-$(config-resource)}" + local node jsonpath + node="$(get-hostname-for-vm)" + jsonpath="{.status.nodes['$node'].status}" + + vm-command "kubectl wait -n kube-system $resource \ + --for=jsonpath=\"$jsonpath\"=\"$status\" --timeout=$timeout" || { + echo "Unexpected configuration status:" + vm-command "kubectl get -n kube-system $resource -o jsonpath=\"{.status}\" | jq ." + error "expected $status configuration status of $resource" + } +} + +verify-config-status-error() { # script API + # Usage: verify-config-status-error REGEXP [RESOURCE] + # + # Fail the test unless REGEXP matches the errors in the node status of the + # configuration custom resource RESOURCE, which defaults to the + # configuration resource of $POLICY. + local regexp="$1" resource="${2:-$(config-resource)}" + + vm-command "kubectl get -n kube-system $resource -ojson | jq '.status.nodes[].errors'" + grep -q "$regexp" <<< "$COMMAND_OUTPUT" || + error "expected an error matching \"$regexp\" in the status of $resource" +} + declare -a pulled_images_on_vm create() { # script API # Usage: [VAR=VALUE][n=COUNT] create TEMPLATE_NAME From 28956cc1249e5bcd5528b6ca3292bba7f370ca55 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:08:31 +0300 Subject: [PATCH 14/31] e2e: add helm-reconfigure and patch-policy-config. test19-pct reconfigures a running plugin in place with a helm upgrade, and to do that it repeats the whole --set argument list of helm-launch. Factor that list out as helm-set-args, so that helm-launch and the new helm-reconfigure cannot drift apart, and drop the copy from the test. test18-turbo-priority patches the configuration custom resource of the policy five times, each time naming the resource explicitly and adding its own error handling. patch-policy-config derives the resource from $POLICY like the other configuration helpers in run.sh. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../n4c16/test18-turbo-priority/code.var.sh | 15 ++--- .../balloons/n4c16/test19-pct/code.var.sh | 22 +------ test/e2e/run.sh | 62 +++++++++++++++++-- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh index 52adbf2d1..5d8d86ca2 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh @@ -450,8 +450,7 @@ fi # changes do not lag one step behind the active config. defaultcls_step3=$(enforce-count) echo "defaultclass step 3: patching cpuClasses default.maxFreq base -> turbo" -vm-command "kubectl -n kube-system patch balloonspolicies/default --type=merge -p '{\"spec\":{\"cpuClasses\":[{\"name\":\"default\",\"minFreq\":\"min\",\"maxFreq\":\"turbo\"},{\"name\":\"fast\",\"minFreq\":\"turbo\",\"maxFreq\":\"turbo\"}]}}'" || - command-error "[defaultclass step3] kubectl patch of balloonspolicies/default failed" +patch-policy-config '{"spec":{"cpuClasses":[{"name":"default","minFreq":"min","maxFreq":"turbo"},{"name":"fast","minFreq":"turbo","maxFreq":"turbo"}]}}' wait-enforce-grows "$defaultcls_step3" echo "defaultclass step 3: $defaultcls_step3 -> $(enforce-count) enforce writes immediately after CR patch" assert-class-written "$defaultcls_step3" "default" "defaultclass step3 default class write after cpuClasses change" @@ -471,8 +470,7 @@ done # applied immediately on every default-class CPU. defaultcls_step4=$(enforce-count) echo "defaultclass step 4: patching cpuClasses default.maxFreq turbo -> base" -vm-command "kubectl -n kube-system patch balloonspolicies/default --type=merge -p '{\"spec\":{\"cpuClasses\":[{\"name\":\"default\",\"minFreq\":\"min\",\"maxFreq\":\"base\"},{\"name\":\"fast\",\"minFreq\":\"turbo\",\"maxFreq\":\"turbo\"}]}}'" || - command-error "[defaultclass step4] kubectl patch of balloonspolicies/default failed" +patch-policy-config '{"spec":{"cpuClasses":[{"name":"default","minFreq":"min","maxFreq":"base"},{"name":"fast","minFreq":"turbo","maxFreq":"turbo"}]}}' wait-enforce-grows "$defaultcls_step4" echo "defaultclass step 4: $defaultcls_step4 -> $(enforce-count) enforce writes immediately after revert patch" assert-class-written "$defaultcls_step4" "default" "defaultclass step4 default class write after cpuClasses revert" @@ -501,8 +499,7 @@ helm-terminate # reserved CPU keeps maxFreq=3800000; under turboDomain=system it drops # to base (2900000) because turbo-high (prio=10) wins globally. helm_config=$TEST_DIR/balloons-turbo.cfg helm-launch balloons -vm-command "kubectl -n kube-system patch balloonspolicies/default --type=merge -p '{\"spec\":{\"availableResources\":{\"cpu\":\"cpuset:2,10\"},\"reservedResources\":{\"cpu\":\"1000m\"}}}'" || - command-error "[turboDomain setup] kubectl patch (cpuset:2,10) failed" +patch-policy-config '{"spec":{"availableResources":{"cpu":"cpuset:2,10"},"reservedResources":{"cpu":"1000m"}}}' wait-enforce-grows 0 td_setup_count=$(enforce-count) echo "turboDomain setup: $td_setup_count enforce writes after cpuset patch" @@ -541,8 +538,7 @@ echo "turboDomain package: cpu $reserved_cpu (default-turbo) at max=3800000 as e # default-turbo loses turbo everywhere, so cpu $reserved_cpu must drop # to base (2900000) immediately after the CR patch. td_sys_before=$(enforce-count) -vm-command "kubectl -n kube-system patch balloonspolicies/default --type=merge -p '{\"spec\":{\"turboDomain\":\"system\"}}'" || - command-error "[turboDomain system] kubectl patch (turboDomain=system) failed" +patch-policy-config '{"spec":{"turboDomain":"system"}}' wait-enforce-grows "$td_sys_before" enforce-lines-since "$td_sys_before" sys_lines="$COMMAND_OUTPUT" @@ -555,8 +551,7 @@ echo "turboDomain system: cpu $reserved_cpu (default-turbo) at max=2900000 as ex # Revert to turboDomain=package: cpu $reserved_cpu must climb back to # 3800000 immediately on CR patch. td_back_before=$(enforce-count) -vm-command "kubectl -n kube-system patch balloonspolicies/default --type=merge -p '{\"spec\":{\"turboDomain\":\"package\"}}'" || - command-error "[turboDomain back] kubectl patch (turboDomain=package) failed" +patch-policy-config '{"spec":{"turboDomain":"package"}}' wait-enforce-grows "$td_back_before" enforce-lines-since "$td_back_before" back_lines="$COMMAND_OUTPUT" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index b16e9708b..ebf404ab8 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -42,24 +42,6 @@ wait-ext-hp() { wait-node-resource "$ext_hp" "$@" } -# helm-upgrade performs an in-process reconfiguration by -# upgrading the running helm release with a new plugin config. Only -# the policy custom resource changes, so the plugin pod is not -# restarted and the agent reconfigures in place. -helm-upgrade() { - local cfg=$1 - host-command "$SCP \"$cfg\" $VM_HOSTNAME:" || - command-error "copying \"$cfg\" to VM failed" - vm-command "helm upgrade -n kube-system test ./helm/balloons \ - --values=$(basename "$cfg") \ - --set image.name=localhost/balloons \ - --set image.tag=testing \ - --set image.pullPolicy=Never \ - --set resources.cpu=50m \ - --set resources.memory=256Mi \ - --set plugin-test.enableAPIs=true" || - command-error "helm upgrade with $cfg failed" -} ############################################################################### @@ -192,12 +174,12 @@ wait-ext-hp 4 "HP extended resource not published after (re)launch" # pct-hp cpuClass but stops publishing it. The running plugin must # reconcile the node and REMOVE the now-unowned resource without a # restart. -helm-upgrade "$TEST_DIR/balloons-pct-nopublish.cfg" +helm-reconfigure balloons "$TEST_DIR/balloons-pct-nopublish.cfg" wait-ext-hp missing "in-process reconfig to a non-publishing config did not remove the HP extended resource" # In-process reconfiguration back to publishing: the resource must # reappear. -helm-upgrade "$TEST_DIR/balloons-pct-managed.cfg" +helm-reconfigure balloons "$TEST_DIR/balloons-pct-managed.cfg" wait-ext-hp 4 "in-process reconfig back to a publishing config did not re-publish the HP extended resource" # Tear down the release so the next helm-launch starts from a clean diff --git a/test/e2e/run.sh b/test/e2e/run.sh index aa4c7b81d..37b55c798 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -412,6 +412,19 @@ instantiate() { # script API echo "$RESULT" } +helm-set-args() { # script API + # Usage: helm-set-args PLUGIN + # + # Print the --set arguments which helm-launch and helm-reconfigure give to + # helm for PLUGIN. + echo "--set image.name=localhost/$1 \ + --set image.tag=testing \ + --set image.pullPolicy=Never \ + --set resources.cpu=50m \ + --set resources.memory=256Mi \ + --set plugin-test.enableAPIs=true" +} + helm-launch() { # script API # Usage: helm-launch TARGET # @@ -458,12 +471,7 @@ helm-launch() { # script API vm-command "helm install $rollback -n kube-system $helm_name ./helm/$plugin \ --values=`basename ${helm_config}` \ - --set image.name=localhost/$plugin \ - --set image.tag=testing \ - --set image.pullPolicy=Never \ - --set resources.cpu=50m \ - --set resources.memory=256Mi \ - --set plugin-test.enableAPIs=true" || + $(helm-set-args "$plugin")" || error "failed to helm install/start plugin $plugin" case "$timeout" in @@ -532,6 +540,33 @@ helm-launch() { # script API vm-port-forward-enable } +helm-reconfigure() { # script API + # Usage: helm-reconfigure PLUGIN CONFIG + # + # Reconfigure a running PLUGIN by upgrading its helm release with the + # configuration helm override values in CONFIG. This updates the + # configuration custom resource without restarting the plugin, so the + # plugin reconfigures itself in place. + # + # Environment variables: + # helm_name: helm installation name to upgrade + # default: test + # + # Example: + # helm-reconfigure balloons $TEST_DIR/balloons-other.cfg + # + local plugin="$1" config="$2" + local helm_name="${helm_name:-test}" + + host-command "$SCP \"$config\" $VM_HOSTNAME:" || + command-error "copying \"$config\" to VM failed" + + vm-command "helm upgrade -n kube-system $helm_name ./helm/$plugin \ + --values=$(basename "$config") \ + $(helm-set-args "$plugin")" || + command-error "helm upgrade of $plugin with $config failed" +} + helm-terminate() { # script API # Usage: helm-terminate # @@ -667,6 +702,21 @@ wait-config-status() { # script API } } +patch-policy-config() { # script API + # Usage: patch-policy-config JSON-MERGE-PATCH [RESOURCE] + # + # Patch the configuration custom resource RESOURCE, which defaults to the + # configuration resource of $POLICY, with a JSON merge patch. Fail the test + # if patching fails. + # + # Example: + # patch-policy-config '{"spec":{"turboDomain":"system"}}' + local patch="$1" resource="${2:-$(config-resource)}" + + vm-command "kubectl -n kube-system patch $resource --type=merge -p '$patch'" || + command-error "patching $resource failed" +} + verify-config-status-error() { # script API # Usage: verify-config-status-error REGEXP [RESOURCE] # From bf6c7c9b97f4b230edf3a43a51888b795cf74bb1 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:11:06 +0300 Subject: [PATCH 15/31] e2e: add CLOS and CPU frequency assertion helpers. Three tests assert that the plugin associated CPUs to a CLOS, and two that it enforced the frequencies of a CPU class, each of them spelling out the log line the plugin writes. That is the log format of the cpu control knowing in eleven places what it should know in one. The rest of the CPU frequency assertions stay in the tests which use them. The enforce write counting and windowing in test18-turbo-priority and the c-state override checks in test17-cstates-scheduling have a single user each, and encode what that test means by a minimal set of writes rather than anything reusable. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 22 ++++++++ .../balloons/n4c16/test19-pct/code.var.sh | 8 +-- .../n4c16/test24-podresources/code.var.sh | 4 +- .../n4c16/test19-cpuclass/code.var.sh | 56 ++++++------------- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 832617282..f62641695 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -522,6 +522,28 @@ wait-assert-log-grew() { # script API command-error "$msg (pattern: $pattern, expected more than $count lines)" } +assert-cpu-clos() { # script API + # Usage: assert-cpu-clos CPUS CLOS [MESSAGE] [TIMEOUT] + # + # Wait until the plugin has associated CPUS to CLOS, for instance + # "CLOS 0". CPUS is an extended regular expression matching a CPU list, so + # ".*" matches any CPUs. + local cpus=$1 clos=$2 msg=${3:-"CPUs '$1' not associated to $2"} tmo=${4:-5} + wait-assert-log-contains "associated cpus $cpus to $clos" "$msg" "$tmo" +} + +assert-cpu-freq() { # script API + # Usage: assert-cpu-freq CPULIST CLASS [MESSAGE] [TIMEOUT] + # + # Wait until the plugin has enforced the CPU frequencies of CPU class + # CLASS on every CPU in CPULIST. + local cpus=$1 class=$2 msg=$3 tmo=${4:-5} cpu + for cpu in $(expand-cpulist "$cpus"); do + wait-assert-log-contains "enforcing cpu frequency from class .$class@.* on cpu $cpu\$" \ + "${msg:-CPU frequency class $class not enforced on cpu $cpu}" "$tmo" + done +} + ### ### Cleaning up ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh index ebf404ab8..06e898b0f 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test19-pct/code.var.sh @@ -67,14 +67,14 @@ CPUREQ=1 CPULIM=1 MEMREQ=10M MEMLIM=10M \ EXTLIM="cpuclass.balloons.nri.io/pct-hp: \"1\"" \ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: pct-hp-bln" CONTCOUNT=1 \ create balloons-busybox -wait-assert-log-contains 'associated cpus .* to CLOS 0' "HP pod CPUs not associated to CLOS 0" +assert-cpu-clos '.*' 'CLOS 0' "HP pod CPUs not associated to CLOS 0" report allowed # Phase 1.3: schedule a pod in the LP balloon. CPUREQ=1 CPULIM=1 MEMREQ=10M MEMLIM=10M \ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: pct-lp-bln" CONTCOUNT=1 \ create balloons-busybox -wait-assert-log-contains 'associated cpus .* to CLOS 3' "LP pod CPUs not associated to CLOS 3" +assert-cpu-clos '.*' 'CLOS 3' "LP pod CPUs not associated to CLOS 3" report allowed # Phase 1.3b: verify HP-reserve allocation steering. The HP balloon @@ -141,7 +141,7 @@ verify 'len(cpus["pod3c0"]) == 2' CPUREQ=1 CPULIM=1 MEMREQ=10M MEMLIM=10M \ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: pct-lp2-bln" CONTCOUNT=1 \ create balloons-busybox -wait-assert-log-contains 'associated cpus .* to CLOS 3' "LP2 pod CPUs not associated to CLOS 3" +assert-cpu-clos '.*' 'CLOS 3' "LP2 pod CPUs not associated to CLOS 3" report allowed # T1.4: per-class idle reassociation. Delete the LP pod (pod1). @@ -249,7 +249,7 @@ CPUREQ=1 CPULIM=1 MEMREQ=10M MEMLIM=10M \ POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: assoc-clos1-bln" CONTCOUNT=1 \ create balloons-busybox report allowed -wait-assert-log-contains 'associated cpus .* to CLOS 1' "CPUs not associated to CLOS 1 in assoc-only mode" +assert-cpu-clos '.*' 'CLOS 1' "CPUs not associated to CLOS 1 in assoc-only mode" # Now that a full pod admission has gone through without any PCT # startup-time configuration, the negative checks for managed-mode diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh index d63bcda7e..e83d6deb9 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh @@ -141,8 +141,8 @@ verify 'disjoint_sets(nodes["pod1c0"], nodes["pod1c1"], nodes["pod1c2"], nodes[" # Sanity check (cf. test19-pct): the hp-near-tpu balloons use the # pct-hp cpuClass, so their CPUs must have been associated to the PCT # high-priority CLOS 0. -plugin-log 'associated cpus .* to CLOS 0' \ - || command-error "hp-near-tpu balloon CPUs were not associated to PCT HP CLOS 0" +assert-cpu-clos '.*' 'CLOS 0' \ + "hp-near-tpu balloon CPUs were not associated to PCT HP CLOS 0" vm-command "kubectl delete pods --all --now" diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh index 31d33f2db..959873383 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test19-cpuclass/code.var.sh @@ -6,28 +6,6 @@ cleanup() { # Restrict the log assertions to the cpuclass-related log lines. plugin_log_filter=' *cpuclass *' -# assert-cpu-clos -# Polls the log until a default timeout to verify that the given CPUs are -# associated to the given CLOS. -assert-cpu-clos() { - local ctr="$1" cpus="$2" clos="$3" - wait-assert-log-contains "associated cpus $cpus to $clos" \ - "Missing CPU ($cpus) association for $ctr (expected to $clos)" -} - -# assert-cpu-freq -# Polls the log until a default timeout to verify that the given CPU's -# frequency is reprogrammed according to the given class. -assert-cpu-freq() { - local user="$1" cpus="$2" class="$3" - local cpu="" - - for cpu in $(expand-cpulist "$cpus"); do - wait-assert-log-contains "enforcing cpu frequency from class .$class@.* on cpu $cpu" \ - "Missing CPU frequency class $class enforcement on cpu $cpu for $user" - done -} - OVERRIDE_SYS_CPUFREQ='[{"cpus": "0-15", "base": 2900000, "min": 800000, "max": 3800000}]' OVERRIDE_SST='{"supported": true, "clos_count": 4, "packages": [{"id": 0, "cpus": "0-7", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}, {"id": 1, "cpus": "8-15", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}]}' OVERRIDE_SST_STATE_DIR="/tmp/nri-pct-mock" @@ -69,7 +47,7 @@ wait-node-resource cpuclass.resource-policy.nri.io/exclusive 4 \ # cpu0=0 -assert-cpu-freq "reserved pool" $cpu0 reserved +assert-cpu-freq $cpu0 reserved # # Default exclusive CPU class assignment @@ -82,21 +60,21 @@ CONTCOUNT=4 CPU=1 create guaranteed pod=pod0 cpu0=$(container-cpus $pod ${pod}c0) -assert-cpu-clos ${pod}c0 $cpu0 "CLOS 0" +assert-cpu-clos $cpu0 "CLOS 0" cpu1=$(container-cpus $pod ${pod}c1) -assert-cpu-clos ${pod}c1 $cpu1 "CLOS 0" +assert-cpu-clos $cpu1 "CLOS 0" cpu2=$(container-cpus $pod ${pod}c2) -assert-cpu-clos ${pod}c2 $cpu2 "CLOS 0" +assert-cpu-clos $cpu2 "CLOS 0" cpu3=$(container-cpus $pod ${pod}c3) -assert-cpu-clos ${pod}c3 $cpu3 "CLOS 0" +assert-cpu-clos $cpu3 "CLOS 0" # Delete pod. Verify that each released exclusive CPU gets assigned to # the shared pool CPU class which is configured with low PCT priority. vm-command "kubectl delete pod $pod" -assert-cpu-clos ${pod}c0 $cpu0 "CLOS 3" -assert-cpu-clos ${pod}c1 $cpu1 "CLOS 3" -assert-cpu-clos ${pod}c2 $cpu2 "CLOS 3" -assert-cpu-clos ${pod}c3 $cpu3 "CLOS 3" +assert-cpu-clos $cpu0 "CLOS 3" +assert-cpu-clos $cpu1 "CLOS 3" +assert-cpu-clos $cpu2 "CLOS 3" +assert-cpu-clos $cpu3 "CLOS 3" # # Container-specific class assignment @@ -112,21 +90,21 @@ ANN1="cpu-class.resource-policy.nri.io/container.${pod}c1: class2" \ CONTCOUNT=4 CPU=2 create guaranteed cpu0=$(container-cpus $pod ${pod}c0) -assert-cpu-freq ${pod}c0 $cpu0 class1 +assert-cpu-freq $cpu0 class1 cpu1=$(container-cpus $pod ${pod}c1) -assert-cpu-freq ${pod}c1 $cpu1 class2 +assert-cpu-freq $cpu1 class2 cpu2=$(container-cpus $pod ${pod}c2) -assert-cpu-clos ${pod}c2 $cpu2 "CLOS 0" +assert-cpu-clos $cpu2 "CLOS 0" cpu3=$(container-cpus $pod ${pod}c3) -assert-cpu-clos ${pod}c3 $cpu3 "CLOS 0" +assert-cpu-clos $cpu3 "CLOS 0" # Delete pod. Verify that each released CPU get assigned to # the shared pool CPU class. vm-command "kubectl delete pod $pod" -assert-cpu-clos ${pod}c0 $cpu0 "CLOS 3" -assert-cpu-clos ${pod}c1 $cpu1 "CLOS 3" -assert-cpu-clos ${pod}c2 $cpu2 "CLOS 3" -assert-cpu-clos ${pod}c3 $cpu3 "CLOS 3" +assert-cpu-clos $cpu0 "CLOS 3" +assert-cpu-clos $cpu1 "CLOS 3" +assert-cpu-clos $cpu2 "CLOS 3" +assert-cpu-clos $cpu3 "CLOS 3" # # Non-eligible container to a CPU class assignment From a6a07d7c0e861ae33e3fc82feb09b470c5190631 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:14:51 +0300 Subject: [PATCH 16/31] e2e: add node reboot, kernel command line and NUMA helpers. test22-isolcpus and test00-basic-placement both set or clear isolcpus on the kernel command line, reboot, verify the result and restart kubelet, and the two test30-numa-disabled tests share their whole NUMA disabling prologue and epilogue apart from the policy name. vm-restart-kubelet waits with wait-for-node-ready instead of waiting for the kube-apiserver process and then for cilium. Waiting for the node to become Ready covers both, and does not assume that the CNI plugin of the VM is cilium, which is only one of the alternatives provisioning supports. The two test30 tests are now identical, and the policy they check for after the reboot comes from $POLICY. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 50 +++++++++++++++++++ test/e2e/lib/vm.bash | 27 ++++++++++ .../n4c16/test22-isolcpus/code.var.sh | 29 +---------- .../n4c16/test30-numa-disabled/code.var.sh | 10 +--- .../n4c16/test00-basic-placement/code.var.sh | 11 +--- .../n4c16/test30-numa-disabled/code.var.sh | 10 +--- 6 files changed, 84 insertions(+), 53 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index f62641695..2e4a07144 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,56 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### Node state +### + +clear-isolcpus() { # script API + # Usage: clear-isolcpus + # + # Remove isolcpus from the kernel command line of the node, rebooting the + # node if it is there. Do nothing if it is not. + # + # A test which isolates CPUs and does not get to restore the command line, + # because it failed, leaves the CPUs isolated. That changes CPU pinning for + # every test which runs after it on the same VM. Call this in a test which + # needs to be sure that no CPUs are isolated. + vm-command "grep isolcpus /proc/cmdline" || return 0 + + vm-set-kernel-cmdline-reboot "" + vm-command "grep isolcpus /proc/cmdline" && + error "failed to remove isolcpus from the kernel command line" + + echo "isolcpus removed from the kernel command line" + return 0 +} + +disable-numa() { # script API + # Usage: disable-numa [POLICY] + # + # Boot the node with a kernel which has NUMA support disabled, and make + # sure that the container runtime and POLICY, $POLICY by default, are + # running afterwards. Do nothing if NUMA is already disabled. + vm-command '[ -d /sys/devices/system/node ]' || return 0 + + vm-kernel-pkgs-install + vm-post-reboot-runtime-check "${1:-$POLICY}" + + vm-command '[ -d /sys/devices/system/node ]' && + error "failed to disable NUMA in the kernel" + return 0 +} + +enable-numa() { # script API + # Usage: enable-numa [POLICY] + # + # Boot the node back with a kernel which has NUMA support, and make sure + # that the container runtime and POLICY, $POLICY by default, are running + # afterwards. + vm-kernel-pkgs-uninstall + vm-post-reboot-runtime-check "${1:-$POLICY}" +} + ### ### CPU lists ### diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index b123fa74e..a877eafb8 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -878,6 +878,33 @@ vm-set-kernel-cmdline() { fi } +vm-restart-kubelet() { # script API + # Usage: vm-restart-kubelet + # + # Restart kubelet and wait until the node is ready again. + vm-command "systemctl restart kubelet" + wait-for-node-ready +} + +vm-set-kernel-cmdline-reboot() { # script API + # Usage: [timeout=SECS] vm-set-kernel-cmdline-reboot [CMDLINE] + # + # Set the kernel command line parameters of the VM to CMDLINE, none by + # default, reboot the VM, and wait until the node is ready again. Verify + # that CMDLINE took effect, and fail the test if it did not. + # + # timeout is the time to wait for the VM to reboot, 600 seconds by default. + local cmdline="$1" + + vm-set-kernel-cmdline "$cmdline" + timeout=${timeout:-600} vm-reboot + if [ -n "$cmdline" ]; then + vm-command "grep -q -- '$cmdline' /proc/cmdline" || + error "failed to set kernel command line parameters \"$cmdline\"" + fi + vm-restart-kubelet +} + vm-kernel-pkgs-install() { # script API # Usage: vm-kernel-pkgs-install # diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh index 0c149e183..3d0a9a665 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh @@ -1,14 +1,3 @@ -reboot-node() { - timeout=600 vm-reboot -} - -restart-kubelet() { - vm-command "systemctl restart kubelet" - sleep 5 - vm-wait-process --timeout 120 kube-apiserver - vm-command "cilium status --wait --wait-duration=120s --interactive=false" -} - cleanup-pods() { delete-pods --all delete-namespaces "$ns" @@ -16,27 +5,13 @@ cleanup-pods() { cleanup() { cleanup-pods - vm-set-kernel-cmdline "" - reboot-node - vm-command "grep -v isolcpus /proc/cmdline" - if [ $? -ne 0 ]; then - error "failed to unset isolcpus kernel commandline parameter" - fi - restart-kubelet - return 0 + clear-isolcpus } ns=isolcpus cleanup-pods -vm-command "grep isolcpus=0,1 /proc/cmdline" || { - vm-set-kernel-cmdline "isolcpus=0,1" - reboot-node - vm-command "grep isolcpus=0,1 /proc/cmdline" || { - error "failed to set isolcpus kernel commandline parameter" - } - restart-kubelet -} +vm-command "grep isolcpus=0,1 /proc/cmdline" || vm-set-kernel-cmdline-reboot "isolcpus=0,1" helm-terminate helm_config=${TEST_DIR}/balloons-isolcpus.cfg helm-launch balloons diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh index 5b5293637..3152e9a13 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh @@ -1,9 +1,4 @@ -vm-command '[ -d /sys/devices/system/node ]' && { - vm-kernel-pkgs-install - vm-post-reboot-runtime-check balloons -} - -vm-command '[ -d /sys/devices/system/node ]' && error "failed to disable NUMA in kernel" +disable-numa helm-terminate helm_config=$TEST_DIR/balloons-numa-disabled.cfg helm-launch balloons @@ -26,5 +21,4 @@ verify "cpus['pod1c0'].isdisjoint({'cpu06', 'cpu07'})" \ "len(cpus['pod1c1']) == 5" \ "disjoint_sets(cpus['pod1c0'], cpus['pod1c1'])" -vm-kernel-pkgs-uninstall -vm-post-reboot-runtime-check balloons +enable-numa diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh index 98703f758..0a7318065 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh @@ -8,16 +8,7 @@ delete-pods -n kube-system pod0 pod1 pod2 pod3 pod4 pod5 # pinning and cause false negatives from other tests on this VM. # This can happen if test08-isolcpus failed and we are re-running # the tests from the start. -vm-command "grep isolcpus /proc/cmdline" && { - vm-set-kernel-cmdline "" - timeout=120 vm-reboot - vm-command "grep isolcpus /proc/cmdline" && { - error "failed to clean up isolcpus kernel commandline parameter" - } - echo "isolcpus removed from kernel commandline" - vm-command "systemctl restart kubelet" - vm-wait-process --timeout 120 kube-apiserver -} +clear-isolcpus # Do a fresh start helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh index 33d82d3fb..ab8292dc6 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test30-numa-disabled/code.var.sh @@ -1,9 +1,4 @@ -vm-command '[ -d /sys/devices/system/node ]' && { - vm-kernel-pkgs-install - vm-post-reboot-runtime-check topology-aware -} - -vm-command '[ -d /sys/devices/system/node ]' && error "failed to disable NUMA in kernel" +disable-numa helm-terminate helm_config=$(instantiate helm-config.yaml) helm-launch topology-aware @@ -21,5 +16,4 @@ verify \ delete-pods --all helm-terminate -vm-kernel-pkgs-uninstall -vm-post-reboot-runtime-check topology-aware +enable-numa From 501b7064f0be674fc1213955ec035496ea7d718a Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:16:29 +0300 Subject: [PATCH 17/31] e2e: move the node resource topology helpers to the shared library. nrt.source.sh was scoped to the balloons policy, so test16-composite-balloons defined a verify-nrt of its own and test15-loadclasses inlined a query for its debug output, both addressing the topology of the node as the first item of a list. Express all three on nrt-query, which addresses the topology of the node the way nrt.source.sh already did. The tests outside n4c16 which use nrt-verify-zone-attribute, nrt-verify-zone-resource and $nrt_kubectl_get need no changes: the library is sourced at the same point of the same chain, only before the *.source.sh files rather than as one of them. Also fix two message bugs which came along: the expected value was missing from the failure message of nrt-verify-zone-attribute, which referred to a variable that never existed, and the resource name was missing from the progress message of nrt-verify-zone-resource. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 47 +++++++++++++++++++ .../n4c16/test15-loadclasses/code.var.sh | 2 +- .../test16-composite-balloons/code.var.sh | 21 ++++----- .../balloons/nrt.source.sh | 24 ---------- 4 files changed, 57 insertions(+), 37 deletions(-) delete mode 100644 test/e2e/policies.test-suite/balloons/nrt.source.sh diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 2e4a07144..b806629f1 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,53 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### Node resource topology +### + +# The kubectl command which addresses the node resource topology of the node. +export nrt_kubectl_get="kubectl get noderesourcetopologies.topology.node.k8s.io \$(hostname)" + +nrt-query() { # script API + # Usage: nrt-query JQ-EXPRESSION + # + # Print the result of evaluating JQ-EXPRESSION on the node resource + # topology of the node, and store it in COMMAND_OUTPUT. + # + # JQ-EXPRESSION must not contain single quotes. + vm-command "$nrt_kubectl_get -o json | jq -r '$1'" +} + +nrt-verify-zone-attribute() { # script API + # Usage: nrt-verify-zone-attribute ZONE ATTRIBUTE REGEXP + # + # Fail the test unless the value of ATTRIBUTE of topology zone ZONE + # matches REGEXP. + local zone_name=$1 + local attribute_name=$2 + local expected_value_re=$3 + echo "" + echo "### Verifying topology zone $zone_name attribute $attribute_name value matches $expected_value_re" + nrt-query ".zones[] | select (.name == \"$zone_name\").attributes[] | select(.name == \"$attribute_name\").value" + [[ "$COMMAND_OUTPUT" =~ $expected_value_re ]] || + command-error "expected zone $zone_name attribute $attribute_name value $expected_value_re, got: $COMMAND_OUTPUT" +} + +nrt-verify-zone-resource() { # script API + # Usage: nrt-verify-zone-resource ZONE RESOURCE FIELD VALUE + # + # Fail the test unless FIELD of RESOURCE of topology zone ZONE equals VALUE. + local zone_name=$1 + local resource_name=$2 + local resource_field=$3 + local expected_value=$4 + echo "" + echo "### Verifying topology zone $zone_name resource $resource_name field $resource_field equals $expected_value" + nrt-query ".zones[] | select (.name == \"$zone_name\").resources[] | select(.name == \"$resource_name\").$resource_field" + [[ "$COMMAND_OUTPUT" == "$expected_value" ]] || + command-error "expected zone $zone_name resource $resource_name.$resource_field $expected_value, got: $COMMAND_OUTPUT" +} + ### ### Node state ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh index c33937ef4..f557cba03 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh @@ -23,7 +23,7 @@ cleanup() { CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="" POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: l2load" CONTCOUNT=2 create balloons-busybox # Print balloons and their cpusets from NRT for debugging. -vm-command 'kubectl get -n kube-system noderesourcetopologies.topology.node.k8s.io -o json | jq ".items[].zones[] | select(.type == \"balloon\") | {balloon:.name, cpuset:(.attributes[] | select(.name == \"cpuset\") | .value)}"' +nrt-query '.zones[] | select(.type == "balloon") | {balloon:.name, cpuset:(.attributes[] | select(.name == "cpuset") | .value)}' report allowed verify 'len(cpus["pod0c0"]) == 1' \ 'len(cpus["pod0c1"]) == 1' \ diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh index 7e080dc12..f38f0f1d8 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh @@ -9,13 +9,10 @@ cleanup() { } verify-nrt() { - jqquery="$1" - expected="$2" - vm-command "kubectl get -n kube-system noderesourcetopologies.topology.node.k8s.io -o json | jq -r '$jqquery'" - if [[ -n "$expected" ]]; then - if [[ "$expected" != "$COMMAND_OUTPUT" ]]; then - command-error "invalid output, expected: '$expected'" - fi + local jqquery="$1" expected="$2" + nrt-query "$jqquery" + if [[ -n "$expected" ]] && [[ "$expected" != "$COMMAND_OUTPUT" ]]; then + command-error "invalid output, expected: '$expected'" fi } @@ -28,8 +25,8 @@ verify 'len(cpus["pod0c0"]) == 4' \ 'len(cpus["pod0c1"]) == 4' \ 'nodes["pod0c0"] == nodes["pod0c1"] == {"node0", "node1", "node2", "node3"}' -verify-nrt '.items[0].zones[] | select (.name == "balance-all-nodes[0]")' # no check, print for debugging -verify-nrt '.items[0].zones[] | select (.name == "balance-all-nodes[0]") .attributes[] | select (.name == "excess cpus") .value' 3000m +verify-nrt '.zones[] | select (.name == "balance-all-nodes[0]")' # no check, print for debugging +verify-nrt '.zones[] | select (.name == "balance-all-nodes[0]") .attributes[] | select (.name == "excess cpus") .value' 3000m # Balance a large workload on all NUMA nodes CPUREQ="9" MEMREQ="100M" CPULIM="" MEMLIM="" @@ -43,7 +40,7 @@ verify 'len(cpus["pod1c0"]) == 12' \ 'len(set.intersection(cpus["pod1c0"], {"cpu12", "cpu13", "cpu14", "cpu15"})) == 3' \ 'len(set.intersection(cpus["pod1c0"], {"cpu06", "cpu07"})) == 1' # cpu06 or cpu07 is reserved -verify-nrt '.items[0].zones[] | select (.name == "balance-all-nodes[0]")' # no check, print for debugging +verify-nrt '.zones[] | select (.name == "balance-all-nodes[0]")' # no check, print for debugging CPUREQ="100m" MEMREQ="" CPULIM="100m" MEMLIM="" namespace=kube-system create balloons-busybox @@ -71,7 +68,7 @@ report allowed verify 'len(cpus["pod3c0"]) == 2' \ 'len(set.intersection(cpus["pod3c0"], {"cpu08", "cpu09", "cpu10", "cpu11"})) == 1' \ 'len(set.intersection(cpus["pod3c0"], {"cpu12", "cpu13", "cpu14", "cpu15"})) == 1' -verify-nrt '.items[0].zones[] | select (.name == "balance-pkg1-nodes[0]")' # no check, print for debugging +verify-nrt '.zones[] | select (.name == "balance-pkg1-nodes[0]")' # no check, print for debugging CPUREQ="4" MEMREQ="100M" CPULIM="" MEMLIM="" POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: balance-pkg1-nodes" CONTCOUNT=1 create balloons-busybox @@ -80,7 +77,7 @@ verify 'len(cpus["pod4c0"]) == 4' \ 'len(set.intersection(cpus["pod4c0"], {"cpu08", "cpu09", "cpu10", "cpu11"})) == 2' \ 'len(set.intersection(cpus["pod4c0"], {"cpu12", "cpu13", "cpu14", "cpu15"})) == 2' \ 'disjoint_sets(cpus["pod4c0"], cpus["pod3c0"])' -verify-nrt '.items[0].zones[] | select (.name == "balance-pkg1-nodes[1]")' # no check, print for debugging +verify-nrt '.zones[] | select (.name == "balance-pkg1-nodes[1]")' # no check, print for debugging # Remove pods. Now composite balloons balance-pkg1-nodes[0] and # balance-pkg1-nodes[1] should be deleted completely (in contrast to diff --git a/test/e2e/policies.test-suite/balloons/nrt.source.sh b/test/e2e/policies.test-suite/balloons/nrt.source.sh deleted file mode 100644 index b47ca9ca2..000000000 --- a/test/e2e/policies.test-suite/balloons/nrt.source.sh +++ /dev/null @@ -1,24 +0,0 @@ -export nrt_kubectl_get="kubectl get noderesourcetopologies.topology.node.k8s.io \$(hostname)" - -nrt-verify-zone-attribute() { - local zone_name=$1 - local attribute_name=$2 - local expected_value_re=$3 - echo "" - echo "### Verifying topology zone $zone_name attribute $attribute_name value matches $expected_value_re" - vm-command "$nrt_kubectl_get -o json | jq -r '.zones[] | select (.name == \"$zone_name\").attributes[] | select(.name == \"$attribute_name\").value'" - [[ "$COMMAND_OUTPUT" =~ $expected_value_re ]] || - command-error "expected zone $zone_name attribute $attribute_name value $expected_value, got: $COMMAND_OUTPUT" -} - -nrt-verify-zone-resource() { - local zone_name=$1 - local resource_name=$2 - local resource_field=$3 - local expected_value=$4 - echo "" - echo "### Verifying topology zone $zone_name resource $resouce_name field $resource_field equals $expected_value" - vm-command "$nrt_kubectl_get -o json | jq -r '.zones[] | select (.name == \"$zone_name\").resources[] | select(.name == \"$resource_name\").$resource_field'" - [[ "$COMMAND_OUTPUT" == "$expected_value" ]] || - command-error "expected zone $zone_name resource $resource_name.$resource_field $expected_value, got: $COMMAND_OUTPUT" -} From 6ddcbdbb2143f0b9d719a61c0f588a903e5b256b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:17:33 +0300 Subject: [PATCH 18/31] e2e: move the metrics helpers to the shared library. verify.source.sh was scoped to the balloons policy although nothing in it is balloons-specific: it just reads the metrics endpoint of the instrumentation server, which any plugin can be configured to open. The URL of that server was also hardcoded in test21-controller-check, which reads another endpoint of it, so give it a name of its own. The curl commands now pass --noproxy localhost, as the ones in test21-controller-check already did. Without it, a run which sets a proxy for the VM tries to reach the plugin through the proxy. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 31 +++++++++++++++++++ .../n4c16/test21-controller-check/code.var.sh | 8 ++--- .../balloons/verify.source.sh | 17 ---------- 3 files changed, 35 insertions(+), 21 deletions(-) delete mode 100644 test/e2e/policies.test-suite/balloons/verify.source.sh diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index b806629f1..458aa9c1f 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -158,6 +158,37 @@ expect-launch-failure() { # script API echo "launching $policy failed as expected" } +### +### Metrics and instrumentation +### + +# The instrumentation HTTP server of the plugin, and its metrics endpoint. +# The plugin exports these only if its configuration opens the server. +instrumentation_url="http://localhost:8891" +verify_metrics_url="$instrumentation_url/metrics" + +verify-metrics-has-line() { # script API + # Usage: verify-metrics-has-line REGEXP [TIMEOUT] + # + # Wait until the metrics of the plugin have a line matching REGEXP, an + # extended regular expression. Fail the test on timeout, 10s by default. + local expected_line="$1" + vm-run-until --timeout "${2:-10}" "echo 'waiting for metrics line: $expected_line' >&2; curl --silent --noproxy localhost $verify_metrics_url | grep -E '$expected_line'" || { + command-error "expected line '$1' missing from the output" + } +} + +verify-metrics-has-no-line() { # script API + # Usage: verify-metrics-has-no-line REGEXP [TIMEOUT] + # + # Wait until the metrics of the plugin have no line matching REGEXP. Fail + # the test on timeout, 10s by default. + local unexpected_line="$1" + vm-run-until --timeout "${2:-10}" "echo 'checking absence of metrics line: $unexpected_line' >&2; ! curl --silent --noproxy localhost $verify_metrics_url | grep -Eq '$unexpected_line'" || { + command-error "unexpected line '$1' found from the output" + } +} + ### ### Node resource topology ### diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh index 86ab84cb0..c15c1773e 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh @@ -5,7 +5,7 @@ helm_config=$TEST_DIR/balloons-config.cfg helm-launch balloons # Check that the test controller starts and gets called in proper places vm-run-until --timeout 5 \ - "curl --silent --noproxy localhost http://localhost:8891/e2e-test-controller-state | \ + "curl --silent --noproxy localhost $instrumentation_url/e2e-test-controller-state | \ jq '.Log.ControllerEvent[]' 2>&1 | grep -q Start" || \ error "Controller not started properly." @@ -13,12 +13,12 @@ vm-run-until --timeout 5 \ CPUREQ="" CPULIM="" MEMREQ="" MEMLIM="" CONTCOUNT=2 create balloons-busybox # For pod creation we should see PreCreate and PostStart events -vm-command-q "curl --silent --noproxy localhost http://localhost:8891/e2e-test-controller-state" | jq '.Log.PreCreate[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PreCreate event not proper" -vm-command-q "curl --silent --noproxy localhost http://localhost:8891/e2e-test-controller-state" | jq '.Log.PostStart[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PostStart event not proper" +vm-command-q "curl --silent --noproxy localhost $instrumentation_url/e2e-test-controller-state" | jq '.Log.PreCreate[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PreCreate event not proper" +vm-command-q "curl --silent --noproxy localhost $instrumentation_url/e2e-test-controller-state" | jq '.Log.PostStart[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PostStart event not proper" # Then delete the pod, we should see PostStop event vm-command "kubectl delete pods pod0 --now" -vm-command-q "curl --silent --noproxy localhost http://localhost:8891/e2e-test-controller-state" | jq '.Log.PostStop[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PostStop event not proper" +vm-command-q "curl --silent --noproxy localhost $instrumentation_url/e2e-test-controller-state" | jq '.Log.PostStop[]' 2>&1 | tr -d '"' | awk -v RS="" '/pod0c0/&&/pod0c1/{r=1; exit} END{exit !r}' || error "PostStop event not proper" helm-terminate diff --git a/test/e2e/policies.test-suite/balloons/verify.source.sh b/test/e2e/policies.test-suite/balloons/verify.source.sh deleted file mode 100644 index e1350ab69..000000000 --- a/test/e2e/policies.test-suite/balloons/verify.source.sh +++ /dev/null @@ -1,17 +0,0 @@ -# Utilities to verify data from metrics - -verify_metrics_url="http://localhost:8891/metrics" - -verify-metrics-has-line() { - local expected_line="$1" - vm-run-until --timeout 10 "echo 'waiting for metrics line: $expected_line' >&2; curl --silent $verify_metrics_url | grep -E '$expected_line'" || { - command-error "expected line '$1' missing from the output" - } -} - -verify-metrics-has-no-line() { - local unexpected_line="$1" - vm-run-until --timeout 10 "echo 'checking absence of metrics line: $unexpected_line' >&2; ! curl --silent $verify_metrics_url | grep -Eq '$unexpected_line'" || { - command-error "unexpected line '$1' found from the output" - } -} From 9a1586704bc50ae5a7701b0f3b70a435d7e59756 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:18:43 +0300 Subject: [PATCH 19/31] e2e: fix pod names in test07-maxballoons error messages. The failure messages of the second allocation failure check refer to pod6, but the pod the check creates is pod5. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/n4c16/test07-maxballoons/code.var.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh index 2e54371f5..91b16c9af 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh @@ -52,11 +52,11 @@ verify 'cpus["pod4c0"] == cpus["pod3c0"]' # pod5: preferring new balloon fails, and fitting to existing dynamictwo balloons fails CPUREQ="300m" CPULIM="300m" ( POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: dynamictwo" CONTCOUNT=1 wait_t=5s create balloons-busybox ) && { - error "creating pod6 succeeded but was expected to fail with balloon allocation error" + error "creating pod5 succeeded but was expected to fail with balloon allocation error" } vm-command "kubectl describe pod pod5" if ! grep -q 'no suitable balloon instance available' <<< "$COMMAND_OUTPUT"; then - error "could not find 'no suitable balloon instance available' in pod6 description" + error "could not find 'no suitable balloon instance available' in pod5 description" fi vm-command "kubectl delete pod pod5 --now" From 711e3d4e18af3ef1896e47b90a99fc191e928b5c Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:20:34 +0300 Subject: [PATCH 20/31] e2e: add relaunch-policy helper. Restarting the plugin with a new configuration is the most repeated pair of commands in the suite. Give it a name. Converted are the 27 sites which name their configuration file, where the one-liner also reads better than the pair it replaces. The sites which build their configuration with instantiate keep the two lines: passing a command substitution through a quoted argument would not make them any clearer. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/test.bash | 21 +++++++++++++++++++ .../n4c16/test01-basic-placement/code.var.sh | 3 +-- .../test02-prometheus-metrics/code.var.sh | 3 +-- .../n4c16/test03-reserved/code.var.sh | 3 +-- .../balloons/n4c16/test04-groupby/code.var.sh | 3 +-- .../n4c16/test05-namespace/code.var.sh | 3 +-- .../n4c16/test06-update-config/code.var.sh | 3 +-- .../n4c16/test07-maxballoons/code.var.sh | 3 +-- .../balloons/n4c16/test08-numa/code.var.sh | 3 +-- .../n4c16/test09-isolated/code.var.sh | 3 +-- .../n4c16/test10-allocator-opts/code.var.sh | 3 +-- .../n4c16/test11-match-expression/code.var.sh | 3 +-- .../n4c16/test13-cacheclusters/code.var.sh | 6 ++---- .../n4c16/test15-loadclasses/code.var.sh | 3 +-- .../test16-composite-balloons/code.var.sh | 3 +-- .../test17-cstates-scheduling/code.var.sh | 3 +-- .../n4c16/test18-turbo-priority/code.var.sh | 3 +-- .../n4c16/test20-config-status/code.var.sh | 3 +-- .../n4c16/test21-controller-check/code.var.sh | 3 +-- .../n4c16/test22-isolcpus/code.var.sh | 3 +-- .../n4c16/test23-available-cpus/code.var.sh | 3 +-- .../n4c16/test24-podresources/code.var.sh | 3 +-- .../balloons/n4c16/test25-irq/code.var.sh | 12 ++++------- .../n4c16/test30-numa-disabled/code.var.sh | 3 +-- 24 files changed, 48 insertions(+), 54 deletions(-) diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 458aa9c1f..ef24f03d3 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -138,6 +138,27 @@ wait-pod-gone() { # script API ### Launching the plugin ### +relaunch-policy() { # script API + # Usage: relaunch-policy POLICY [CONFIG] + # + # Terminate the plugin if one is running, then launch POLICY with the + # configuration helm override values in CONFIG. Without CONFIG, launch it + # with the configuration helm-launch uses by default. + # + # Honour the same environment variables as helm-launch. + # + # Example: + # relaunch-policy balloons "$TEST_DIR/balloons-reserved.cfg" + local policy=$1 config=$2 + + helm-terminate + if [ -n "$config" ]; then + helm_config="$config" helm-launch "$policy" + else + helm-launch "$policy" + fi +} + expect-launch-failure() { # script API # Usage: expect-launch-failure POLICY [TIMEOUT] # diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh index 7c05f730e..6121d4ca5 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh @@ -1,8 +1,7 @@ # Test placing containers with and without annotations to correct balloons # reserved and shared CPUs. -helm-terminate -helm_config=${TEST_DIR}/../../helm-config.yaml helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/../../helm-config.yaml" cleanup() { delete-pods -n kube-system pod0 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh index 199884755..639fb4f33 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh @@ -8,8 +8,7 @@ cleanup # Launch nri-resource-policy with wanted metrics update interval and a # configuration that opens the instrumentation http server. -helm-terminate -helm_config=${TEST_DIR}/balloons-metrics.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-metrics.cfg" verify-metrics-has-line 'balloon="default\[0\]"' verify-metrics-has-line 'balloon="reserved\[0\]"' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh index bd2fff22a..c2004648c 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=${TEST_DIR}/balloons-reserved.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-reserved.cfg" cleanup() { delete-pods -n kube-system pod0 pod3 pod7 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh index 5ceffaa53..e0a3f04a9 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test04-groupby/code.var.sh @@ -1,8 +1,7 @@ # This test verifies that the groupby expression in a balloon type # affects grouping containers into balloon instances of that type. -helm-terminate -helm_config=$TEST_DIR/balloons-groupby.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-groupby.cfg" testns=e2e-balloons-test04 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh index 734d2d707..dd17d6aee 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=${TEST_DIR}/balloons-namespace.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-namespace.cfg" cleanup() { delete-pods --all diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh index 8284a398b..1c064ac39 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test06-update-config/code.var.sh @@ -1,8 +1,7 @@ # This test verifies that configuration updates via nri-resource-policy-agent # are handled properly in the balloons policy. -helm-terminate -helm_config=$TEST_DIR/initial-balloons-config.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/initial-balloons-config.cfg" testns=e2e-balloons-test06 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh index 91b16c9af..a9c86f8e2 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh @@ -4,8 +4,7 @@ cleanup() { cleanup -helm-terminate -helm_config=${TEST_DIR}/balloons-maxballoons.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-maxballoons.cfg" # pod0: allocate 1500/2000 mCPUs of the singleton balloon CPUREQ="1500m" CPULIM="1500m" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test08-numa/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test08-numa/code.var.sh index 8c60d1515..c43a4a05d 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test08-numa/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test08-numa/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=${TEST_DIR}/balloons-numa.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-numa.cfg" # pod0: besteffort, make sure it still gets at least 1 CPU CPUREQ="" CPULIM="" MEMREQ="" MEMLIM="" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/code.var.sh index 4d9fe449e..05e49571e 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=${TEST_DIR}/balloons-isolated.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-isolated.cfg" verify-metrics-has-line 'balloon="isolated-pods\[0\]"' verify-metrics-has-line 'balloon="isolated-pods\[1\]"' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh index e93d084f0..f679fe988 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh @@ -6,8 +6,7 @@ cleanup # Launch cri-resmgr with wanted metrics update interval and a # configuration that opens the instrumentation http server. -helm-terminate -helm_config=${TEST_DIR}/balloons-allocator-opts.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-allocator-opts.cfg" # pod0 in a 2-CPU balloon CPUREQ="100m" MEMREQ="100M" CPULIM="100m" MEMLIM="100M" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh index 8ddb15398..1783e7ebb 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh @@ -1,8 +1,7 @@ # Test placing containers with and without annotations to correct balloons # reserved and shared CPUs. -helm-terminate -helm_config=${TEST_DIR}/../../match-config.yaml helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/../../match-config.yaml" cleanup() { delete-pods --all diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh index 6af65f891..4edb6e4a0 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test13-cacheclusters/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=$TEST_DIR/balloons-4cpu-cacheclusters.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-4cpu-cacheclusters.cfg" cleanup() { delete-pods --all @@ -56,8 +55,7 @@ verify 'nodes["pod3c0"] == nodes["pod2c0"] == nodes["pod2c1"] == nodes["pod2c2"] cleanup -helm-terminate -helm_config=$TEST_DIR/balloons-2cpu-cacheclusters.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-2cpu-cacheclusters.cfg" # pod4c{0,1,2,3}: one container per free L2 group, this time L2 groups contain only single CPU cores CPUREQ="500m" MEMREQ="100M" CPULIM="500m" MEMLIM="100M" diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh index f557cba03..3df5448f3 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test15-loadclasses/code.var.sh @@ -2,8 +2,7 @@ # them. Test that CPU allocation avoids overloading any part of the # system. -helm-terminate -helm_config=$TEST_DIR/balloons-loadclasses.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-loadclasses.cfg" cleanup() { delete-pods --all diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh index f38f0f1d8..f7c12e0c5 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test16-composite-balloons/code.var.sh @@ -1,7 +1,6 @@ # Test balloons that are composed of other balloons. -helm-terminate -helm_config=$TEST_DIR/balloons-composite.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-composite.cfg" cleanup() { delete-pods -n kube-system pod2 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index 4ab6ae759..4e2f06039 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -1,7 +1,6 @@ # Test balloons with certain CPU c-states disabled -helm-terminate -helm_config=$TEST_DIR/balloons-cstates.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-cstates.cfg" # verify-cstates checks the last writes to "disable" files in the # override fs. diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh index 5d8d86ca2..009d8b28b 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test18-turbo-priority/code.var.sh @@ -12,8 +12,7 @@ # *same* turbo-low balloon as pod0) does not produce any new # enforce writes. -helm-terminate -helm_config=$TEST_DIR/balloons-turbo.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-turbo.cfg" # Restrict the log assertions to the turbo recalculation log lines. plugin_log_filter='turbo:|cpuClass' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh index 37ef2c35d..a412bb110 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test20-config-status/code.var.sh @@ -1,5 +1,4 @@ -helm-terminate -helm_config=$TEST_DIR/balloons.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons.cfg" sleep 1 diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh index c15c1773e..57af063cc 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test21-controller-check/code.var.sh @@ -1,7 +1,6 @@ # Test that the nri-resource-policy controllers are called properly -helm-terminate -helm_config=$TEST_DIR/balloons-config.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-config.cfg" # Check that the test controller starts and gets called in proper places vm-run-until --timeout 5 \ diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh index 3d0a9a665..391ce1a13 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test22-isolcpus/code.var.sh @@ -13,8 +13,7 @@ cleanup-pods vm-command "grep isolcpus=0,1 /proc/cmdline" || vm-set-kernel-cmdline-reboot "isolcpus=0,1" -helm-terminate -helm_config=${TEST_DIR}/balloons-isolcpus.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-isolcpus.cfg" create-namespaces "$ns" # pod0: should run on non-isolated CPUs diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh index e01d7ffe9..607ff72d5 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test23-available-cpus/code.var.sh @@ -6,8 +6,7 @@ cleanup() { cleanup create-namespaces reserved -helm-terminate -helm_config=${TEST_DIR}/balloons-excluded-cpusets.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-excluded-cpusets.cfg" # pod0: run on reserved CPUs CPUREQ="50m" CPULIM="" namespace=kube-system CONTCOUNT=1 create balloons-busybox diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh index e83d6deb9..4169f415a 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test24-podresources/code.var.sh @@ -1,8 +1,7 @@ # Test CPU affinity to devices published by device plugins and queried # from kubelet's PodResourcesAPI. -helm-terminate -helm_config=$TEST_DIR/balloons-podresources.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-podresources.cfg" cleanup() { vm-command 'pidof fake-device-plugin && kill $(pidof fake-device-plugin) && sleep 1' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh index b0307c706..e89589fa6 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -16,8 +16,7 @@ cleanup # Test irqClaim. CPUs of the claimer balloon handle the claimed # IRQs (ttyS0 and rtc0). -helm-terminate -helm_config=${TEST_DIR}/balloons-irq-claim.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-irq-claim.cfg" POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox report allowed @@ -31,8 +30,7 @@ verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" # Test irqMode isolate. CPUs of the isolate balloon are removed from # the affinity of unclaimed IRQs. Preset a full affinity so that the # removal is observable. -helm-terminate -helm_config=${TEST_DIR}/balloons-irq-isolate.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-irq-isolate.cfg" cleanup POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: isolate" CONTCOUNT=1 create balloons-busybox @@ -51,8 +49,7 @@ verify-irq-cpus "$RTC0_IRQ" "$expected_isolate" # Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. -helm-terminate -helm_config=${TEST_DIR}/balloons-irq-sink.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-irq-sink.cfg" cleanup # no sink, no claimer present @@ -84,8 +81,7 @@ verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" # Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. -helm-terminate -helm_config=${TEST_DIR}/balloons-irq-dedicated-claim.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-irq-dedicated-claim.cfg" cleanup POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: dedicated-claimer" CONTCOUNT=1 create balloons-busybox diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh index 3152e9a13..5a77df410 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test30-numa-disabled/code.var.sh @@ -1,7 +1,6 @@ disable-numa -helm-terminate -helm_config=$TEST_DIR/balloons-numa-disabled.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-numa-disabled.cfg" POD_ANNOTATION=( "balloon.balloons.resource-policy.nri.io: single-thread-own-core" From bf3d247e9a5e5e925305f0205ac19bcaab5b1c07 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:21:30 +0300 Subject: [PATCH 21/31] e2e: document the shared test helpers. List the helper groups of lib/test.bash in the README, so that the next test does not have to grep the library to find out what is already there, and note the variables which configure them. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index f71d1c6e2..f26f73ee6 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -104,13 +104,31 @@ A test case is a `code.var.sh` file in a `run.sh` (run `./run.sh help` to list it), test cases have shared helpers available: -- `lib/test.bash` contains helpers that are useful for more than one test case, - for instance for waiting for a condition, cleaning up pods and namespaces, - inspecting container states, or reading the log of the plugin. Add a helper - here if a second test case needs it. `./run.sh help` documents these, too. - -- `TEST-SUITE/POLICY/*.source.sh` files contain policy-specific helpers, for - instance `policies.test-suite/balloons/nrt.source.sh`. +- `lib/test.bash` contains helpers that are useful for more than one test case: + + | group | helpers | + | --- | --- | + | waiting | `retry-until` | + | pods and containers | `container-state`, `wait-container-waiting-reason`, `verify-container-error`, `wait-pod-gone` | + | launching the plugin | `relaunch-policy`, `expect-launch-failure` | + | cleaning up | `delete-pods`, `create-namespaces`, `delete-namespaces`, `remove-policy-cache`, `kill-test-processes` | + | log of the plugin | `plugin-daemonset`, `plugin-log`, `plugin-log-tail`, `assert-log-contains`, `assert-log-not-contains`, `wait-assert-log-contains`, `wait-assert-log-grew`, `assert-cpu-clos`, `assert-cpu-freq` | + | metrics | `verify-metrics-has-line`, `verify-metrics-has-no-line` | + | node resource topology | `nrt-query`, `nrt-verify-zone-attribute`, `nrt-verify-zone-resource` | + | node state | `clear-isolcpus`, `disable-numa`, `enable-numa` | + | CPU lists | `expand-cpulist`, `cpulist-difference`, `container-cpus`, `allowed-cpu-ids` | + | extended resources | `get-node-resource`, `wait-node-resource` | + | interrupts | `resolve-irq`, `irq-cpu-ids`, `verify-irq-cpus` | + | scheduling | `verify-sched`, `SCHED_*` | + + Some of them are configured by variables a test case can set, notably + `plugin_log_filter`, which restricts the log assertions to the log lines of + the subsystem under test. + + Add a helper here once a second test case needs it. Run `./run.sh help` for + the documentation of each of them. + +- `TEST-SUITE/POLICY/*.source.sh` files contain policy-specific helpers. `*.source.sh` files are sourced before the test case code, starting from the test suite directory and ending in the test case directory, `lib/test.bash` From 23e3318c9618a861a6ed7b92aa07eddc88bca2dc Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:29:14 +0300 Subject: [PATCH 22/31] e2e: drop the unused cleanup function of test17-cstates-scheduling. The test defines cleanup but never calls it. It deletes both of its pods as part of the scenario, so calling it at the end would be a no-op anyway. Note that the test also never terminates the plugin it launched. That is left as it is, since every test starts by terminating a previously running one. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/n4c16/test17-cstates-scheduling/code.var.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index 4e2f06039..5bdeca6d5 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -39,10 +39,6 @@ verify-cstates-no-writes() { } } -cleanup() { - delete-pods --all -} - echo "verify that all c-states of all available CPUs are enabled" verify-cstates "2 3 4 5 6 7 11 12 13" "C1E C2 C4 C8" "" 40 From e2e18d31704da8e9eec7ee84286596305a950a59 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:41:47 +0300 Subject: [PATCH 23/31] e2e: use the cleanup helpers in the n6-hbm-cxl tests. All four non-fuzz tests of the two n6-hbm-cxl collections start with the same cleanup function, and test01-memory-types with the same terminate and launch pair, as the n4c16 tests did. The awk based pod selection of test02-fuzz-memallocs is not expressible with the helpers and is left as it is, like its n4c16 counterpart. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/n6-hbm-cxl/test01-memory-types/code.var.sh | 6 ++---- .../balloons/n6-hbm-cxl/test04-nrt/code.var.sh | 2 +- .../n6-hbm-cxl/test01-memory-types/code.var.sh | 2 +- .../topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh index 3459e8687..090a899da 100644 --- a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh @@ -1,9 +1,7 @@ -helm-terminate -helm_config=${TEST_DIR}/balloons-memory-types.cfg helm-launch balloons +relaunch-policy balloons "${TEST_DIR}/balloons-memory-types.cfg" cleanup() { - vm-command "kubectl delete pods --all --now" - return 0 + delete-pods --all } cleanup diff --git a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh index 59a873209..8ea0f75a1 100644 --- a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh @@ -1,5 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate } diff --git a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh index 8b4644a2b..b205f8813 100644 --- a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh @@ -1,5 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate } diff --git a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh index ae3297609..66656532c 100644 --- a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh @@ -1,5 +1,5 @@ cleanup() { - vm-command "kubectl delete pods --all --now" + delete-pods --all helm-terminate } From 41ef1f367ee388dc0b402277129e0a39d04070a9 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:45:08 +0300 Subject: [PATCH 24/31] e2e: use the shared NRT helpers in the n6-hbm-cxl tests. topology-aware/test04-nrt carried its own copy of the kubectl command and of the zone attribute assertion, predating the shared ones. The two copies differ in that the shared assertion matches a regular expression while this one compared for equality, so anchor the patterns of the converted call sites to keep the comparisons exact. Unanchored, three of them would have accepted a superset: "3" would match a reserved cpuset of "13" or "3-5". The regular expression is what the balloons test needs, where the expected value is a genuine alternation of the cpusets the policy may pick, so the two cannot be merged the other way around. Both tests print the topology for debugging, four times in total, which is now nrt-dump. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 2 +- test/e2e/lib/test.bash | 7 +++++ .../n6-hbm-cxl/test04-nrt/code.var.sh | 6 ++-- .../n6-hbm-cxl/test04-nrt/code.var.sh | 29 ++++++------------- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index f26f73ee6..0a45f43f1 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -114,7 +114,7 @@ available: | cleaning up | `delete-pods`, `create-namespaces`, `delete-namespaces`, `remove-policy-cache`, `kill-test-processes` | | log of the plugin | `plugin-daemonset`, `plugin-log`, `plugin-log-tail`, `assert-log-contains`, `assert-log-not-contains`, `wait-assert-log-contains`, `wait-assert-log-grew`, `assert-cpu-clos`, `assert-cpu-freq` | | metrics | `verify-metrics-has-line`, `verify-metrics-has-no-line` | - | node resource topology | `nrt-query`, `nrt-verify-zone-attribute`, `nrt-verify-zone-resource` | + | node resource topology | `nrt-query`, `nrt-dump`, `nrt-verify-zone-attribute`, `nrt-verify-zone-resource` | | node state | `clear-isolcpus`, `disable-numa`, `enable-numa` | | CPU lists | `expand-cpulist`, `cpulist-difference`, `container-cpus`, `allowed-cpu-ids` | | extended resources | `get-node-resource`, `wait-node-resource` | diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index ef24f03d3..4484c28fe 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -227,6 +227,13 @@ nrt-query() { # script API vm-command "$nrt_kubectl_get -o json | jq -r '$1'" } +nrt-dump() { # script API + # Usage: nrt-dump + # + # Print the node resource topology of the node as YAML, for debugging. + vm-command "$nrt_kubectl_get -o yaml" +} + nrt-verify-zone-attribute() { # script API # Usage: nrt-verify-zone-attribute ZONE ATTRIBUTE REGEXP # diff --git a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh index 8ea0f75a1..3461ed258 100644 --- a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh @@ -7,7 +7,7 @@ cleanup helm_config=${TEST_DIR}/balloons-nrt.cfg helm-launch balloons # Print full NRT yaml for debugging -vm-command "$nrt_kubectl_get -o yaml" +nrt-dump # Verify zones when fullsocket balloons do not include containers. nrt-verify-zone-attribute "fullsocket[0]" "cpuset" "[4567]" @@ -33,7 +33,7 @@ POD_ANNOTATION='cpu.preserve.resource-policy.nri.io/container.pod0c1: "true" CONTCOUNT=3 create balloons-busybox # Print full NRT yaml for debugging -vm-command "$nrt_kubectl_get -o yaml" +nrt-dump # Verify selected zone attributes nrt-verify-zone-resource "default/pod0/pod0c0" "cpu" "capacity" "4" # balloon's + shared CPUs @@ -54,7 +54,7 @@ POD_ANNOTATION='' CONTCOUNT=2 create balloons-busybox # Print full NRT yaml for debugging -vm-command "$nrt_kubectl_get -o yaml" +nrt-dump nrt-verify-zone-resource "default/pod1/pod1c0" "cpu" "capacity" "1500m" # limit < allowed cpus nrt-verify-zone-attribute "default/pod1/pod1c0" "cpuset" "0-2" # expected fullsocket[1] diff --git a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh index 66656532c..8add5d2fd 100644 --- a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test04-nrt/code.var.sh @@ -6,28 +6,17 @@ cleanup() { cleanup helm_config=${TEST_DIR}/topology-aware-nrt.cfg helm-launch topology-aware -get_nrt="kubectl get noderesourcetopologies.topology.node.k8s.io \$(hostname)" - -verify-zone-attribute() { - local zone_name=$1 - local attribute_name=$2 - local expected_value=$3 - vm-command "$get_nrt -o json | jq -r '.zones[] | select (.name == \"$zone_name\").attributes[] | select(.name == \"$attribute_name\").value'" - [[ "$COMMAND_OUTPUT" == "$expected_value" ]] || - command-error "expected zone $zone_name attribute $attribute_name value $expected_value, got: $COMMAND_OUTPUT" -} - # Print full NRT yaml for debugging -vm-command "$get_nrt -o yaml" +nrt-dump # Verify selected zone attributes -verify-zone-attribute "socket #0" "memory set" "0,2,4" -verify-zone-attribute "socket #0" "shared cpuset" "0-2" -verify-zone-attribute "socket #0" "reserved cpuset" "3" +nrt-verify-zone-attribute "socket #0" "memory set" '^0,2,4$' +nrt-verify-zone-attribute "socket #0" "shared cpuset" '^0-2$' +nrt-verify-zone-attribute "socket #0" "reserved cpuset" '^3$' -verify-zone-attribute "socket #1" "memory set" "1,3,5" -verify-zone-attribute "socket #1" "shared cpuset" "4-7" +nrt-verify-zone-attribute "socket #1" "memory set" '^1,3,5$' +nrt-verify-zone-attribute "socket #1" "shared cpuset" '^4-7$' -verify-zone-attribute "root" "memory set" "0-5" -verify-zone-attribute "root" "shared cpuset" "0-2,4-7" -verify-zone-attribute "root" "reserved cpuset" "3" +nrt-verify-zone-attribute "root" "memory set" '^0-5$' +nrt-verify-zone-attribute "root" "shared cpuset" '^0-2,4-7$' +nrt-verify-zone-attribute "root" "reserved cpuset" '^3$' From 91014680eb70c980ba6ae4e1e3d6f86d831c7e9f Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:49:03 +0300 Subject: [PATCH 25/31] e2e: distil the memory type verifications of the n6-hbm-cxl tests. Both test01-memory-types tests verify that a container uses a given set of memory types, and both spell out the same conditional twenty-one times in total: compare the memory nodes of the container against the nodes of package 0, or against those of package 1 if the container does not run in package 0. local_mems() states the same thing as the memory types it expects, which is what the annotation under test asks for, and keeps the mapping from a memory type to a memory node in one place per topology. It also reports what went wrong. The conditional only ever evaluated to False, leaving it to the reader of the log to work out which package the container ended up in and which nodes that means. Test suite level py_consts.var.py is a new file. run_tests.sh appends py_consts across all levels, so the helper lands before the topology level definitions it uses, which is fine as Python resolves them when the helper runs. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/n6-hbm-cxl/py_consts.var.py | 7 ++++++ .../test01-memory-types/code.var.sh | 16 ++++++------- test/e2e/policies.test-suite/py_consts.var.py | 24 +++++++++++++++++++ .../n6-hbm-cxl/py_consts.var.py | 7 ++++++ .../test01-memory-types/code.var.sh | 24 +++++++++---------- 5 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 test/e2e/policies.test-suite/py_consts.var.py diff --git a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/py_consts.var.py b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/py_consts.var.py index 5571e9d1c..029d70d8b 100644 --- a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/py_consts.var.py +++ b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/py_consts.var.py @@ -4,3 +4,10 @@ hbm1 = "node3" pmem0 = "node4" pmem1 = "node5" + +# Which memory node of a package holds which memory type. local_mems() of the +# test suite level py_consts.var.py uses this. +memory_nodes = { + 0: {"dram": dram0, "hbm": hbm0, "pmem": pmem0}, + 1: {"dram": dram1, "hbm": hbm1, "pmem": pmem1}, +} diff --git a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh index 090a899da..802b784ce 100644 --- a/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test01-memory-types/code.var.sh @@ -24,14 +24,14 @@ POD_ANNOTATION[17]="balloon.balloons.resource-policy.nri.io/container.pod0c7: no POD_ANNOTATION[18]="balloon.balloons.resource-policy.nri.io/container.pod0c8: no-pin-mem" CPUREQ="200m" MEMREQ="300M" CPULIM="" MEMLIM="300M" CONTCOUNT=9 create balloons-busybox report allowed -verify 'mems["pod0c0"] == {hbm0} if packages["pod0c0"] == {"package0"} else mems["pod0c0"] == {hbm1}' \ - 'mems["pod0c1"] == {dram0} if packages["pod0c1"] == {"package0"} else mems["pod0c1"] == {dram1}' \ - 'mems["pod0c2"] == {pmem0} if packages["pod0c2"] == {"package0"} else mems["pod0c2"] == {pmem1}' \ - 'mems["pod0c3"] == {hbm0,dram0} if packages["pod0c3"] == {"package0"} else mems["pod0c3"] == {hbm1,dram1}' \ - 'mems["pod0c4"] == {dram0,pmem0} if packages["pod0c4"] == {"package0"} else mems["pod0c4"] == {dram1,pmem1}' \ - 'mems["pod0c5"] == {hbm0,dram0,pmem0} if packages["pod0c5"] == {"package0"} else mems["pod0c5"] == {hbm1,dram1,pmem1}' \ - 'mems["pod0c6"] == {hbm0,pmem0} if packages["pod0c6"] == {"package0"} else mems["pod0c6"] == {hbm1,pmem1}' \ - 'mems["pod0c7"] == {dram0} if packages["pod0c7"] == {"package0"} else mems["pod0c7"] == {dram1}' \ +verify 'local_mems("pod0c0", "hbm")' \ + 'local_mems("pod0c1", "dram")' \ + 'local_mems("pod0c2", "pmem")' \ + 'local_mems("pod0c3", "hbm", "dram")' \ + 'local_mems("pod0c4", "dram", "pmem")' \ + 'local_mems("pod0c5", "hbm", "dram", "pmem")' \ + 'local_mems("pod0c6", "hbm", "pmem")' \ + 'local_mems("pod0c7", "dram")' \ 'mems["pod0c8"] == {dram0,dram1,hbm0,hbm1,pmem0,pmem1}' cleanup diff --git a/test/e2e/policies.test-suite/py_consts.var.py b/test/e2e/policies.test-suite/py_consts.var.py new file mode 100644 index 000000000..c4c2f4e8d --- /dev/null +++ b/test/e2e/policies.test-suite/py_consts.var.py @@ -0,0 +1,24 @@ +# Helpers for the expressions which "verify" evaluates, shared by every policy +# and topology of this test suite. +# +# These run after the state which "report allowed" collects, so they can refer +# to its variables: cpus, mems, nodes, cores, threads, dies, packages and +# allocations. See "run.sh help pyexec". + +def local_mems(ctr, *types): + """Verify that ctr uses exactly the given memory types of its own package. + + The memory_nodes mapping tells which memory node of a package holds which + memory type. A topology which has more than one memory type defines it, + see for instance n6-hbm-cxl/py_consts.var.py. + """ + package = 0 if packages[ctr] == {"package0"} else 1 + expected = set(memory_nodes[package][t] for t in types) + assert mems[ctr] == expected, ( + "%s runs in %s, so its %s memory should be %s, but it uses %s" % + (ctr, + ",".join(sorted(packages[ctr])), + ",".join(types), + ",".join(sorted(expected)), + ",".join(sorted(mems[ctr])))) + return True diff --git a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/py_consts.var.py b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/py_consts.var.py index 5571e9d1c..029d70d8b 100644 --- a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/py_consts.var.py +++ b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/py_consts.var.py @@ -4,3 +4,10 @@ hbm1 = "node3" pmem0 = "node4" pmem1 = "node5" + +# Which memory node of a package holds which memory type. local_mems() of the +# test suite level py_consts.var.py uses this. +memory_nodes = { + 0: {"dram": dram0, "hbm": hbm0, "pmem": pmem0}, + 1: {"dram": dram1, "hbm": hbm1, "pmem": pmem1}, +} diff --git a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh index b205f8813..c596dd73c 100644 --- a/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n6-hbm-cxl/test01-memory-types/code.var.sh @@ -16,10 +16,10 @@ ANN1="memory-type.resource-policy.nri.io/container.pod0c1: dram" \ create guaranteed report allowed -verify 'mems["pod0c0"] == {dram0, pmem0} if packages["pod0c0"] == {"package0"} else mems["pod0c0"] == {dram1, pmem1}' \ - 'mems["pod0c1"] == {dram0} if packages["pod0c1"] == {"package0"} else mems["pod0c1"] == {dram1}' \ - 'mems["pod0c2"] == {hbm0} if packages["pod0c2"] == {"package0"} else mems["pod0c2"] == {hbm1}' \ - 'mems["pod0c3"] == {pmem0} if packages["pod0c3"] == {"package0"} else mems["pod0c3"] == {pmem1}' +verify 'local_mems("pod0c0", "dram", "pmem")' \ + 'local_mems("pod0c1", "dram")' \ + 'local_mems("pod0c2", "hbm")' \ + 'local_mems("pod0c3", "pmem")' # Release memory allocated for pod0c*. If something is left behind in # hbm or dram, the next text fails. If not, it will @@ -35,10 +35,10 @@ ANN0="memory-type.resource-policy.nri.io/container.pod1c0: hbm,dram" \ create guaranteed report allowed -verify 'mems["pod1c0"] == {hbm0, dram0} if packages["pod1c0"] == {"package0"} else mems["pod1c0"] == {hbm1, dram1}' \ - 'mems["pod1c1"] == {hbm0, dram0} if packages["pod1c1"] == {"package0"} else mems["pod1c1"] == {hbm1, dram1}' \ - 'mems["pod1c2"] == {pmem0} if packages["pod1c2"] == {"package0"} else mems["pod1c2"] == {pmem1}' \ - 'mems["pod1c3"] == {pmem0} if packages["pod1c3"] == {"package0"} else mems["pod1c3"] == {pmem1}' +verify 'local_mems("pod1c0", "hbm", "dram")' \ + 'local_mems("pod1c1", "hbm", "dram")' \ + 'local_mems("pod1c2", "pmem")' \ + 'local_mems("pod1c3", "pmem")' # 2.6G + 2.6G of PMEM is consumed, 1.4G + 1.4G remains. One more 2.0G # pmem allocation does not fit into any single PMEM node. libmem will @@ -56,10 +56,10 @@ ANN0="memory-type.resource-policy.nri.io/container.pod2c0: pmem" \ create guaranteed report allowed -verify 'mems["pod1c0"] == {hbm0, dram0} if packages["pod1c0"] == {"package0"} else mems["pod1c0"] == {hbm1, dram1}' \ - 'mems["pod1c1"] == {hbm0, dram0} if packages["pod1c1"] == {"package0"} else mems["pod1c1"] == {hbm1, dram1}' \ - 'mems["pod1c2"] == {pmem0} if packages["pod1c2"] == {"package0"} else mems["pod1c2"] == {pmem1}' \ - 'mems["pod1c3"] == {pmem0} if packages["pod1c3"] == {"package0"} else mems["pod1c3"] == {pmem1}' \ +verify 'local_mems("pod1c0", "hbm", "dram")' \ + 'local_mems("pod1c1", "hbm", "dram")' \ + 'local_mems("pod1c2", "pmem")' \ + 'local_mems("pod1c3", "pmem")' \ 'mems["pod2c0"] == {pmem0, pmem1}' cleanup From 2c2b5a330cf143271081e128f6a57560198b2326 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:56:39 +0300 Subject: [PATCH 26/31] e2e: add CPU hot-plug helpers. Both s8c4k tests open with a byte-identical 39 line prologue which prepares the sparse 4k CPU topology. Start distilling it by lifting out the sysfs details of hot-plugging. vm-cpus-enabled replaces a grep for the literal "511,1535,4095" in the list of enabled CPUs, which only worked because the three CPUs happen to be listed next to each other. It expands the list and checks each CPU separately, so it also works for CPUs which the kernel reports as part of a range. Onlining the hot-plugged CPUs is now followed by vm-restart-kubelet, which also waits for the node to become ready, instead of restarting kubelet without waiting for anything. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/lib/vm.bash | 42 +++++++++++++++++++ .../s8c4k/test01-sparse-4kcpus/code.var.sh | 20 ++++----- .../s8c4k/test01-sparse-4kcpus/code.var.sh | 20 ++++----- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index a877eafb8..746b62251 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -878,6 +878,48 @@ vm-set-kernel-cmdline() { fi } +vm-cpus-enabled() { # script API + # Usage: vm-cpus-enabled CPU... + # + # Return success if all the given CPUs are enabled in the VM, that is, + # they are either present from the start or have been hot-plugged. + # + # Example: + # vm-cpus-enabled 511 1535 || vm-cpu-hotplug ... + local enabled cpu + vm-command "cat /sys/devices/system/cpu/enabled" + enabled=$(expand-cpulist "$(tr -d '[:space:]' <<< "$COMMAND_OUTPUT")") + for cpu in "$@"; do + [[ " $enabled " == *" $cpu "* ]] || return 1 + done + return 0 +} + +vm-wait-cpus() { # script API + # Usage: vm-wait-cpus CPU... + # + # Wait until the kernel of the VM has exposed all the given CPUs in sysfs. + # Use this after hot-plugging CPUs. + local cpu test="" + for cpu in "$@"; do + test="${test}${test:+ && }[ -d /sys/devices/system/cpu/cpu$cpu ]" + done + vm-run-until "$test" +} + +vm-online-all-cpus() { # script API + # Usage: vm-online-all-cpus + # + # Bring every offline CPU of the VM online. Print the resulting state of + # each CPU. Never fails: a CPU which cannot be onlined is reported but + # tolerated, as not all of them can be. + vm-command 'for cpuX in /sys/devices/system/cpu/cpu[1-9]*; do + echo onlining $cpuX + ( echo 1 > $cpuX/online && echo Successful: write 1 to $cpuX/online ) || echo Failed: write 1 to $cpuX/online + done + grep . /sys/devices/system/cpu/cpu[1-9]*/online' +} + vm-restart-kubelet() { # script API # Usage: vm-restart-kubelet # diff --git a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh index 23565ecc2..a2c000bc2 100644 --- a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -5,24 +5,18 @@ if [ "$( ( echo $min_kernel_version; echo $COMMAND_OUTPUT ) | sort --version-sor error "quest OS runs too old kernel, hot-plugged CPU node topology may not work. Required: $min_kernel_version" fi -# Hot-plug CPUs. -vm-command 'grep 511,1535,4095 /sys/devices/system/cpu/enabled' || { +# Hot-plug core 511 of sockets 0, 2 and 7. With 512 cores per socket, those +# cores are cpu511, cpu1535 and cpu4095. +vm-cpus-enabled 511 1535 4095 || { vm-cpu-hotplug 0 511 0 vm-cpu-hotplug 2 511 0 vm-cpu-hotplug 7 511 0 - # Wait for the kernel to expose all hot-plugged CPUs in sysfs. - vm-run-until '[ -d /sys/devices/system/cpu/cpu511 ] && [ -d /sys/devices/system/cpu/cpu1535 ] && [ -d /sys/devices/system/cpu/cpu4095 ]' + vm-wait-cpus 511 1535 4095 + vm-online-all-cpus - # Online all CPUs. - vm-command 'for cpuX in /sys/devices/system/cpu/cpu[1-9]*; do - echo onlining $cpuX - ( echo 1 > $cpuX/online && echo Successful: write 1 to $cpuX/online ) || echo Failed: write 1 to $cpuX/online - done - grep . /sys/devices/system/cpu/cpu[1-9]*/online' - - # Restart kubelet to let it detect new enabled CPUs. - vm-command "systemctl restart kubelet" + # Restart kubelet to let it detect the new enabled CPUs. + vm-restart-kubelet } # Wait until kubelet has reported all enabled CPUs in node capacity. diff --git a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh index a070d0d0a..4da17fdec 100644 --- a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -5,24 +5,18 @@ if [ "$( ( echo $min_kernel_version; echo $COMMAND_OUTPUT ) | sort --version-sor error "quest OS runs too old kernel, hot-plugged CPU node topology may not work. Required: $min_kernel_version" fi -# Hot-plug CPUs. -vm-command 'grep 511,1535,4095 /sys/devices/system/cpu/enabled' || { +# Hot-plug core 511 of sockets 0, 2 and 7. With 512 cores per socket, those +# cores are cpu511, cpu1535 and cpu4095. +vm-cpus-enabled 511 1535 4095 || { vm-cpu-hotplug 0 511 0 vm-cpu-hotplug 2 511 0 vm-cpu-hotplug 7 511 0 - # Wait for the kernel to expose all hot-plugged CPUs in sysfs. - vm-run-until '[ -d /sys/devices/system/cpu/cpu511 ] && [ -d /sys/devices/system/cpu/cpu1535 ] && [ -d /sys/devices/system/cpu/cpu4095 ]' + vm-wait-cpus 511 1535 4095 + vm-online-all-cpus - # Online all CPUs. - vm-command 'for cpuX in /sys/devices/system/cpu/cpu[1-9]*; do - echo onlining $cpuX - ( echo 1 > $cpuX/online && echo Successful: write 1 to $cpuX/online ) || echo Failed: write 1 to $cpuX/online - done - grep . /sys/devices/system/cpu/cpu[1-9]*/online' - - # Restart kubelet to let it detect new enabled CPUs. - vm-command "systemctl restart kubelet" + # Restart kubelet to let it detect the new enabled CPUs. + vm-restart-kubelet } # Wait until kubelet has reported all enabled CPUs in node capacity. From 40c4ff02de665792a4f95a0f5c0dbd40fe2a57ed Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:58:35 +0300 Subject: [PATCH 27/31] e2e: add kernel version and kubepods cpuset check helpers. The rest of the shared prologue of the two s8c4k tests: a kernel version requirement, waiting for kubelet to report the hot-plugged CPUs in the capacity of the node, and a check that the root cgroup of the pods can use them. Together with the hot-plug helpers this shrinks the prologue from 39 lines to 22. Waiting for the capacity needs no helper of its own, wait-node-resource does it. That also makes the comparison exact: the old check grepped the capacity for a 6, which a capacity of 16 or 64 would have satisfied too. verify-kubepods-cpus likewise expands the cpuset instead of grepping it for each CPU as a substring, so a cpuset of 40950 no longer counts as having CPU 4095. It handles both the single and the multiple cpuset file cases, which differ in whether grep prefixes its output with file names. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- test/e2e/README.md | 2 +- test/e2e/lib/test.bash | 48 +++++++++++++++++-- .../s8c4k/test01-sparse-4kcpus/code.var.sh | 18 ++----- .../s8c4k/test01-sparse-4kcpus/code.var.sh | 20 ++------ 4 files changed, 54 insertions(+), 34 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 0a45f43f1..37e227c02 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -115,7 +115,7 @@ available: | log of the plugin | `plugin-daemonset`, `plugin-log`, `plugin-log-tail`, `assert-log-contains`, `assert-log-not-contains`, `wait-assert-log-contains`, `wait-assert-log-grew`, `assert-cpu-clos`, `assert-cpu-freq` | | metrics | `verify-metrics-has-line`, `verify-metrics-has-no-line` | | node resource topology | `nrt-query`, `nrt-dump`, `nrt-verify-zone-attribute`, `nrt-verify-zone-resource` | - | node state | `clear-isolcpus`, `disable-numa`, `enable-numa` | + | node state | `require-kernel-version`, `verify-kubepods-cpus`, `clear-isolcpus`, `disable-numa`, `enable-numa` | | CPU lists | `expand-cpulist`, `cpulist-difference`, `container-cpus`, `allowed-cpu-ids` | | extended resources | `get-node-resource`, `wait-node-resource` | | interrupts | `resolve-irq`, `irq-cpu-ids`, `verify-irq-cpus` | diff --git a/test/e2e/lib/test.bash b/test/e2e/lib/test.bash index 4484c28fe..a164f0293 100644 --- a/test/e2e/lib/test.bash +++ b/test/e2e/lib/test.bash @@ -268,6 +268,46 @@ nrt-verify-zone-resource() { # script API ### Node state ### +require-kernel-version() { # script API + # Usage: require-kernel-version VERSION [REASON] + # + # Fail the test unless the kernel of the VM is newer than VERSION. REASON + # tells why the test needs it. Note that a kernel whose version equals + # VERSION counts as too old. + # + # Example: + # require-kernel-version 6.14 "CPU hot-plug may not work" + local required=$1 reason=$2 newest + + vm-command "uname -r" + newest=$( ( echo "$required"; echo "$COMMAND_OUTPUT" ) | sort --version-sort | tail -n 1 ) + if [ "$newest" == "$required" ]; then + error "the guest kernel is older than $required${reason:+: $reason}" + fi +} + +verify-kubepods-cpus() { # script API + # Usage: verify-kubepods-cpus CPU... + # + # Fail the test unless the cpuset of the kubepods cgroup, that is, the root + # cgroup of all pods, contains all the given CPUs. + local cpu line cpus="" + + # containerd puts the pods in kubepods, cri-o in kubepods.slice. + vm-command "grep . /sys/fs/cgroup/kubepods*/cpuset.cpus" + + # With more than one match grep prefixes the lines with the file name. + while read -r line; do + [ -n "$line" ] || continue + cpus="$cpus $(expand-cpulist "$(sed 's/.*://' <<< "$line" | tr -d '[:space:]')")" + done <<< "$COMMAND_OUTPUT" + + for cpu in "$@"; do + [[ " $cpus " == *" $cpu "* ]] || + command-error "cpu $cpu is missing from the kubepods cpuset.cpus" + done +} + clear-isolcpus() { # script API # Usage: clear-isolcpus # @@ -508,7 +548,9 @@ get-node-resource() { # script API # Usage: get-node-resource [--allocatable] NAME # # Print the capacity, or with --allocatable the allocatable amount, of - # extended resource NAME on the test node, and store it in COMMAND_OUTPUT. + # resource NAME on the test node, and store it in COMMAND_OUTPUT. NAME is + # an extended resource such as cpuclass.balloons.nri.io/pct-hp, or a + # standard one such as cpu or memory. # Print "missing" if the node does not have the resource at all. local field=capacity while [ "${1#--}" != "$1" ]; do @@ -524,8 +566,8 @@ get-node-resource() { # script API wait-node-resource() { # script API # Usage: wait-node-resource [--allocatable] [--timeout SECS] [--interval SECS] NAME VALUE [MESSAGE] # - # Wait until extended resource NAME on the test node equals VALUE, which - # can also be the string "missing". Give up after SECS seconds, 30 by + # Wait until resource NAME on the test node equals VALUE, which can also + # be the string "missing". Give up after SECS seconds, 30 by # default, checking every SECS seconds, 2 by default. Fail the test with # MESSAGE on timeout. local fieldopt="" tmo=30 ival=2 diff --git a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh index a2c000bc2..b7d221e10 100644 --- a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -1,9 +1,5 @@ -# Prepare virtual machine before installing balloons. -min_kernel_version=6.14 -vm-command "uname -r" -if [ "$( ( echo $min_kernel_version; echo $COMMAND_OUTPUT ) | sort --version-sort | tail -n 1 )" == "$min_kernel_version" ]; then - error "quest OS runs too old kernel, hot-plugged CPU node topology may not work. Required: $min_kernel_version" -fi +# Prepare virtual machine before installing the policy. +require-kernel-version 6.14 "hot-plugged CPU node topology may not work" # Hot-plug core 511 of sockets 0, 2 and 7. With 512 cores per socket, those # cores are cpu511, cpu1535 and cpu4095. @@ -20,16 +16,10 @@ vm-cpus-enabled 511 1535 4095 || { } # Wait until kubelet has reported all enabled CPUs in node capacity. -vm-run-until 'kubectl get node -o jsonpath="{.items[0].status.capacity.cpu}" | grep 6' || - command-error "Unexpected node CPU capacity" +wait-node-resource cpu 6 "kubelet did not report all enabled CPUs in node capacity" # Make sure that k8s root cpuset.cpus contains hot-plugged CPUs. -vm-command "grep . /sys/fs/cgroup/kubepods*/cpuset.cpus" -if ! ( grep -q 511 <<< $COMMAND_OUTPUT && - grep -q 1535 <<< $COMMAND_OUTPUT && - grep -q 4095 <<< $COMMAND_OUTPUT ); then - command-error "kubepods cpuset.cpus does not include expected CPUs" -fi +verify-kubepods-cpus 511 1535 4095 # Install balloons helm-terminate diff --git a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh index 4da17fdec..6438e2fe4 100644 --- a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -1,9 +1,5 @@ -# Prepare virtual machine before installing balloons. -min_kernel_version=6.14 -vm-command "uname -r" -if [ "$( ( echo $min_kernel_version; echo $COMMAND_OUTPUT ) | sort --version-sort | tail -n 1 )" == "$min_kernel_version" ]; then - error "quest OS runs too old kernel, hot-plugged CPU node topology may not work. Required: $min_kernel_version" -fi +# Prepare virtual machine before installing the policy. +require-kernel-version 6.14 "hot-plugged CPU node topology may not work" # Hot-plug core 511 of sockets 0, 2 and 7. With 512 cores per socket, those # cores are cpu511, cpu1535 and cpu4095. @@ -20,18 +16,10 @@ vm-cpus-enabled 511 1535 4095 || { } # Wait until kubelet has reported all enabled CPUs in node capacity. -vm-run-until 'kubectl get node -o jsonpath="{.items[0].status.capacity.cpu}" | grep 6' || - command-error "Unexpected node CPU capacity" +wait-node-resource cpu 6 "kubelet did not report all enabled CPUs in node capacity" # Make sure that k8s root cpuset.cpus contains hot-plugged CPUs. -# containerd: kubepods/cpuset.cpus -# cri-o: kubepods.slice/cpuset.cpus -vm-command "grep . /sys/fs/cgroup/kubepods*/cpuset.cpus" -if ! ( grep -q 511 <<< $COMMAND_OUTPUT && - grep -q 1535 <<< $COMMAND_OUTPUT && - grep -q 4095 <<< $COMMAND_OUTPUT ); then - command-error "kubepods cpuset.cpus does not include expected CPUs" -fi +verify-kubepods-cpus 511 1535 4095 # Install topology-aware helm-terminate From 4213501c33947628089d031baa78094cf581e814 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 16:59:19 +0300 Subject: [PATCH 28/31] e2e: use the existing shared helpers in the s8c4k tests. relaunch-policy, wait-pod-gone and delete-pods, for the same reasons as in the other test collections. The two pod deletions which are steps of the topology-aware scenario, not teardown, are left as they are: they should keep failing if the pod is not there. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/s8c4k/test01-sparse-4kcpus/code.var.sh | 3 +-- .../topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh index b7d221e10..d7cbfb8e5 100644 --- a/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -22,8 +22,7 @@ wait-node-resource cpu 6 "kubelet did not report all enabled CPUs in node capaci verify-kubepods-cpus 511 1535 4095 # Install balloons -helm-terminate -helm_config=$TEST_DIR/balloons-sparse-4kcpus.cfg helm-launch balloons +relaunch-policy balloons "$TEST_DIR/balloons-sparse-4kcpus.cfg" # Verify NRT nrt-verify-zone-resource "reserved[0]" "cpu" "capacity" "6" diff --git a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh index 6438e2fe4..880ff4bad 100644 --- a/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/s8c4k/test01-sparse-4kcpus/code.var.sh @@ -50,7 +50,7 @@ fi # Release all 5 CPUs. vm-command 'kubectl delete pod pod0 --now' -vm-run-until '! kubectl get pod pod0' +wait-pod-gone pod0 # Only socket #0 has enough CPUs for pod1. CONTCOUNT=1 CPUREQ=3100m CPULIM=4000m create burstable @@ -71,4 +71,4 @@ CPU=5000m create guaranteed report allowed verify 'len(cpus["pod4c0"]) == 5' -vm-command 'kubectl delete pods --all --now' +delete-pods --all From 2a68a995cf23264d11f2b34b1a037839456882be Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 17:03:07 +0300 Subject: [PATCH 29/31] e2e: use the shared helpers in the n4c128 test. The same cleanup function and the same terminate and launch pair as everywhere else. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../n4c128/test19-cacheclusters/code.var.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh index 620ec5e85..6d2393090 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh @@ -1,13 +1,11 @@ cleanup() { - vm-command "kubectl delete pods --all --now" - vm-command "kubectl delete namespaces highprio lowprio --now --ignore-not-found" + delete-pods --all + delete-namespaces highprio lowprio } cleanup -helm-terminate - -helm_config=$TEST_DIR/helm-config.yaml helm-launch topology-aware +relaunch-policy topology-aware "$TEST_DIR/helm-config.yaml" # Limit burstability of a container to an L3 cache group and verify that # it gets confined to an L3 cache group. From 03111b34ecc6b20cc2980c7d434cca8fb11b403b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 17:05:53 +0300 Subject: [PATCH 30/31] e2e: distil the repeated loops of the n4c128 test. The test fills every L3 cache with a burstable pod three times over, and each time spells out the same loop and the same pair of assertions about the result. The 127 usable CPUs of the node were a magic number in three places. The helpers stay in the test. They are specific to what it does, and no other test needs them. The annotation itself stays spelled out at the call sites which set it per container. Annotation keys are the interface under test, and keeping them literal is what makes it possible to find the tests which cover a given annotation. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../n4c128/test19-cacheclusters/code.var.sh | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh index 6d2393090..6b95fb036 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh @@ -4,6 +4,26 @@ cleanup() { delete-namespaces highprio lowprio } +# fill-l3caches +# Create one burstable pod per L3 cache, pod to pod, each of them +# limited to bursting within its own L3 cache. +fill-l3caches() { + local i + for i in $(seq "$1" "$2"); do + ANN0="unlimited-burstable.resource-policy.nri.io/container.pod${i}c0: l3cache" + CONTCOUNT=1 CPUREQ=$3 CPULIM=0 MEMREQ=50M create burstable + done +} + +# verify-l3caches-filled +# Verify that the containers of pod to pod occupy all 127 usable +# CPUs of the node in disjoint sets, that is, one full L3 cache each. +verify-l3caches-filled() { + local pods="[cpus[f'pod{i}c0'] for i in range($1, $2 + 1)]" + verify "len(set.union(*$pods)) == 127" \ + "disjoint_sets(*$pods)" +} + cleanup relaunch-policy topology-aware "$TEST_DIR/helm-config.yaml" @@ -108,19 +128,14 @@ cleanup # Fill all 16 L3 caches: 15 with high CPU usage (4 CPUs), 1 with low usage (1 CPU). # The 17th container should be placed in the least occupied L3 cache. # Create 15 pods requesting 4 CPUs each (high occupancy). -for i in $(seq 10 24); do - ANN0="unlimited-burstable.resource-policy.nri.io/container.pod${i}c0: l3cache" - CONTCOUNT=1 CPUREQ=4 CPULIM=0 MEMREQ=50M create burstable -done +fill-l3caches 10 24 4 # Create 1 pod requesting only 1 CPU (low occupancy) - this is the least occupied L3. ANN0='unlimited-burstable.resource-policy.nri.io/container.pod25c0: l3cache' CONTCOUNT=1 CPUREQ=1 CPULIM=0 MEMREQ=50M create burstable report allowed # Verify all 16 L3 caches are occupied and disjoint. -verify \ - 'len(set.union(*[cpus[f"pod{i}c0"] for i in range(10, 26)])) == 127' \ - 'disjoint_sets(*[cpus[f"pod{i}c0"] for i in range(10, 26)])' +verify-l3caches-filled 10 25 # Now add the 17th container - it should share the L3 cache with pod25c0 # (the least occupied one with only 1 CPU used). @@ -137,16 +152,11 @@ cleanup # Fill all 16 L3 caches with 7 CPUs each (leaving only 1 CPU free per cache). # The 17th container requesting 3 CPUs cannot fit in any L3 cache and should # be promoted to the next topology level (NUMA node). -for i in $(seq 27 42); do - ANN0="unlimited-burstable.resource-policy.nri.io/container.pod${i}c0: l3cache" - CONTCOUNT=1 CPUREQ=7 CPULIM=0 MEMREQ=50M create burstable -done +fill-l3caches 27 42 7 report allowed # Verify all 16 L3 caches are occupied with disjoint CPU sets. -verify \ - 'len(set.union(*[cpus[f"pod{i}c0"] for i in range(27, 43)])) == 127' \ - 'disjoint_sets(*[cpus[f"pod{i}c0"] for i in range(27, 43)])' +verify-l3caches-filled 27 42 # Now add the 17th container requesting 3 CPUs - it cannot fit in any L3 cache # (each has only 1 CPU free), so it should be promoted to NUMA level. @@ -169,16 +179,11 @@ cleanup # Test guaranteed pod taking exclusive CPUs from L3 cache shared by burstable pod. # Fill all 16 L3 caches with burstable pods, then create a guaranteed pod. # The guaranteed pod takes exclusive CPUs, reducing the burstable pod's shared CPUs. -for i in $(seq 45 60); do - ANN0="unlimited-burstable.resource-policy.nri.io/container.pod${i}c0: l3cache" - CONTCOUNT=1 CPUREQ=2 CPULIM=0 MEMREQ=50M create burstable -done +fill-l3caches 45 60 2 report allowed # Verify all 16 L3 caches occupied with 8 CPUs each (disjoint). -verify \ - 'len(set.union(*[cpus[f"pod{i}c0"] for i in range(45, 61)])) == 127' \ - 'disjoint_sets(*[cpus[f"pod{i}c0"] for i in range(45, 61)])' +verify-l3caches-filled 45 60 # pod45c0 should have 8 CPUs (full L3 cache). verify 'len(cpus["pod45c0"]) == 8' From a3c525af35926da662dc3efc58dc91e6c8837511 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 26 Aug 2026 17:06:30 +0300 Subject: [PATCH 31/31] e2e: stop deleting namespaces the n4c128 test never creates. The cleanup function came from test18-strict-alignment, which in turn copied it from test17-scheduling-classes, the only test which creates the highprio and lowprio namespaces. In n4c16 the deletion still has a point: test17 and test18 run on the same VM, so test18 cleans up after a test17 which failed before its own cleanup. There is no such test in the n4c128 collection, and no balloons n4c128 collection to share the VM with, so nothing ever creates those namespaces there. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/n4c128/test19-cacheclusters/code.var.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh index 6b95fb036..8893499c0 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c128/test19-cacheclusters/code.var.sh @@ -1,7 +1,6 @@ cleanup() { delete-pods --all - delete-namespaces highprio lowprio } # fill-l3caches