From d64982db95e1b8b2dbc10873f3933c27ce92b542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 15:19:02 +0200 Subject: [PATCH 01/42] combine builds for different accelerators --- bot/build.sh | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 5c9f1af0..dc07ea10 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -161,16 +161,6 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" -# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} -ACCEL_OVERRIDE=$(cfg_get_value "architecture" "accelerator") -if [[ -n "$ACCEL_OVERRIDE" ]]; then - # bot job config does not include accel subdirectory - export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" -else - export EESSI_ACCELERATOR_TARGET_OVERRIDE="" -fi -echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'" - # Log the full lscpu and os-release info: lscpu > _bot_job${SLURM_JOB_ID}.lscpu cat /etc/os-release > _bot_job${SLURM_JOB_ID}.os @@ -259,11 +249,33 @@ fi # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) -echo "Executing command to build software:" -echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" -$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") +if [[ -n "$ACCEL_OVERRIDE" ]]; then + for accel_override in ${ACCEL_OVERRIDES//,/ } + do + # bot job config does not include accel subdirectory + export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" + echo "Executing command to build software:" + echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" + echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} + + # determine temporary directory to resume from for the next accelerator, + # as we want to combine all accelerator builds into a single tarball in the end + BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) + BUILD_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") + done +else + export EESSI_ACCELERATOR_TARGET_OVERRIDE="" + echo "Executing command to build software:" + echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" + echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +fi +echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'" # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step @@ -291,14 +303,15 @@ else fi # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults -# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we # append the project (subdirectory) name to the end tarball name. This is information # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is # appended if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + #export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} accel ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball From 8b113045004b337e3db3ae0a582734c0ff1d780a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 15:41:25 +0200 Subject: [PATCH 02/42] split accelerators by + --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index dc07ea10..d9f8b603 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -252,7 +252,7 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") if [[ -n "$ACCEL_OVERRIDE" ]]; then - for accel_override in ${ACCEL_OVERRIDES//,/ } + for accel_override in ${ACCEL_OVERRIDES//+/ } do # bot job config does not include accel subdirectory export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" From 6f97503cad0c559af0bb419d55e85876df953bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 15:43:11 +0200 Subject: [PATCH 03/42] add easystack --- .../2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb diff --git a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb new file mode 100644 index 00000000..66a4a1f9 --- /dev/null +++ b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb @@ -0,0 +1,2 @@ +easyconfigs: + - pmt-1.3.1-GCCcore-13.3.0-CUDA-12.6.0.eb From bbc6c66e571955b80b2be497bf2202536d830211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 15:54:18 +0200 Subject: [PATCH 04/42] correct file extension --- .../{eessi-2025.06-eb-5.3.0.eb => eessi-2025.06-eb-5.3.0.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2025.06/accel/nvidia/{eessi-2025.06-eb-5.3.0.eb => eessi-2025.06-eb-5.3.0.yml} (100%) diff --git a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml similarity index 100% rename from easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.eb rename to easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml From 41a45d7c899af19962a02577f73f423d47b4ed7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 15:57:37 +0200 Subject: [PATCH 05/42] fix variable name --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index d9f8b603..914dfd9e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -251,7 +251,7 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") -if [[ -n "$ACCEL_OVERRIDE" ]]; then +if [[ -n "$ACCEL_OVERRIDES" ]]; then for accel_override in ${ACCEL_OVERRIDES//+/ } do # bot job config does not include accel subdirectory From 1d2cbed9676b97194b8ade871b5ece1321393c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:04:42 +0200 Subject: [PATCH 06/42] fix another var name --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 914dfd9e..08af5423 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -252,7 +252,7 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") if [[ -n "$ACCEL_OVERRIDES" ]]; then - for accel_override in ${ACCEL_OVERRIDES//+/ } + for ACCEL_OVERRIDE in ${ACCEL_OVERRIDES//+/ } do # bot job config does not include accel subdirectory export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" From 5f02c6c91d78a0c08436abc4ad2101953b74def3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:10:02 +0200 Subject: [PATCH 07/42] add echo --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 08af5423..93541e2f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -255,7 +255,8 @@ if [[ -n "$ACCEL_OVERRIDES" ]]; then for ACCEL_OVERRIDE in ${ACCEL_OVERRIDES//+/ } do # bot job config does not include accel subdirectory - export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" + export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" + echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" echo "Executing command to build software:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" From f2f4587f1982c0853a7f67ec2d03a688d9ec7dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:17:09 +0200 Subject: [PATCH 08/42] use array --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 93541e2f..65ec9b8e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -252,7 +252,8 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") if [[ -n "$ACCEL_OVERRIDES" ]]; then - for ACCEL_OVERRIDE in ${ACCEL_OVERRIDES//+/ } + IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" + for ACCEL_OVERRIDE in "${ACCEL_OVERRIDES_ARRAY[@]}"; do do # bot job config does not include accel subdirectory export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" From 29a419d4037ee7036f07833593819fee30e0fe1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:21:41 +0200 Subject: [PATCH 09/42] remove second do --- bot/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 65ec9b8e..30d950ec 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -254,7 +254,6 @@ ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") if [[ -n "$ACCEL_OVERRIDES" ]]; then IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" for ACCEL_OVERRIDE in "${ACCEL_OVERRIDES_ARRAY[@]}"; do - do # bot job config does not include accel subdirectory export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" From a1d1e913a5915540f442d720ac7c1eba60c15230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:36:38 +0200 Subject: [PATCH 10/42] try passing all accel subdirs to create_tarball script --- bot/build.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 30d950ec..d036bd68 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -251,12 +251,15 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") +IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" +EESSI_ACCELERATOR_TARGET_OVERRIDE=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") if [[ -n "$ACCEL_OVERRIDES" ]]; then - IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" for ACCEL_OVERRIDE in "${ACCEL_OVERRIDES_ARRAY[@]}"; do # bot job config does not include accel subdirectory - export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" - echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" + #export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" + #echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" + export EESSI_ACCELERATOR_TARGET_OVERRIDE="${ACCEL_OVERRIDE}" + echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${ACCEL_OVERRIDE}'" echo "Executing command to build software:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" @@ -328,6 +331,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${ACCEL_OVERRIDES_ARRAY[@]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From f585d447454334128156793693b78ce07c64fe6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:41:41 +0200 Subject: [PATCH 11/42] use new vars --- bot/build.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d036bd68..ddb22f79 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -252,9 +252,10 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" -EESSI_ACCELERATOR_TARGET_OVERRIDE=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") +# prepend accel/ to all array elements +EESSI_ACCELERATOR_TARGET_OVERRIDES=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") if [[ -n "$ACCEL_OVERRIDES" ]]; then - for ACCEL_OVERRIDE in "${ACCEL_OVERRIDES_ARRAY[@]}"; do + for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do # bot job config does not include accel subdirectory #export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" #echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" @@ -331,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${ACCEL_OVERRIDES_ARRAY[@]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From 06ba06dbc5f6a949d68b30150db897da1765ea73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 16:46:36 +0200 Subject: [PATCH 12/42] fix comment --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index ddb22f79..c65c4ddf 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -330,7 +330,7 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} From cee46efed85c3ae3c5cbd2bda7f054dad6121a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 17:01:10 +0200 Subject: [PATCH 13/42] use quotes for create_tarball --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index c65c4ddf..8be534a0 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From d1eb9792995bea3277aae997a20700542d76c11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 20:23:33 +0200 Subject: [PATCH 14/42] pass list of accelerators to create_tarball --- bot/build.sh | 2 +- create_tarball.sh | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8be534a0..2de7ded1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index d3251ba1..3867a992 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -11,7 +11,7 @@ fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 -accel_subdir=$4 +accel_subdirs=$4 target_tarball=$5 tmpdir=`mktemp -d` @@ -64,8 +64,12 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} -if [ -n "${accel_subdir}" ]; then - sw_subdirs="${sw_subdirs} ${cpu_arch_subdir}/${accel_subdir}" +if [ -n "${accel_subdirs}" ]; then + # convert accel_subdirs into an array... + read -r -a accel_subdirs <<< ${accel_subdirs} + # and prepend the cpu_arch_subdir to all elements + accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" + sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" fi for subdir in ${sw_subdirs}; do From e532c96a26418fd543371143ba852e3282315495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 20:40:23 +0200 Subject: [PATCH 15/42] use newlines as delimiter --- bot/build.sh | 2 +- create_tarball.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 2de7ded1..750f1966 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$(printf '%s\n' \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\")" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index 3867a992..0d5a192b 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -66,7 +66,8 @@ fi sw_subdirs=${cpu_arch_subdir} if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... - read -r -a accel_subdirs <<< ${accel_subdirs} + #read -r -a accel_subdirs <<< ${accel_subdirs} + mapfile -t accel_subdirs <<< "$accel_subdirs" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" From 90f01bae6f63a127efedc320ed28c94049f31410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 21:01:28 +0200 Subject: [PATCH 16/42] use eval --- create_tarball.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0d5a192b..c23bd834 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -11,7 +11,7 @@ fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 -accel_subdirs=$4 +accel_subdirs="$4" target_tarball=$5 tmpdir=`mktemp -d` @@ -67,7 +67,8 @@ sw_subdirs=${cpu_arch_subdir} if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... #read -r -a accel_subdirs <<< ${accel_subdirs} - mapfile -t accel_subdirs <<< "$accel_subdirs" + #mapfile -t accel_subdirs <<< "$accel_subdirs" + eval "accel_subdirs=(${accel_subdirs})" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" From 27050bdaa14ae6a6c357c9b365023d8ce3e12ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 23:40:51 +0200 Subject: [PATCH 17/42] debug --- create_tarball.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index c23bd834..5f8dc8aa 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,17 +64,21 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} +echo "accel_subdirs 1: ${accel_subdirs}" if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... #read -r -a accel_subdirs <<< ${accel_subdirs} #mapfile -t accel_subdirs <<< "$accel_subdirs" eval "accel_subdirs=(${accel_subdirs})" + echo "accel_subdirs 2: ${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" + echo "accel_subdirs 3: ${accel_subdirs}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" fi for subdir in ${sw_subdirs}; do + echo "subdir: ${subdir}" if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${subdir}/modules -type f \! -name '.wh.*' >> ${files_list} From d034d0de9637f3efa29698f56dd0320a81fe612e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Apr 2026 23:55:51 +0200 Subject: [PATCH 18/42] use mapfile --- create_tarball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 5f8dc8aa..745bcb04 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -68,8 +68,8 @@ echo "accel_subdirs 1: ${accel_subdirs}" if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... #read -r -a accel_subdirs <<< ${accel_subdirs} - #mapfile -t accel_subdirs <<< "$accel_subdirs" - eval "accel_subdirs=(${accel_subdirs})" + mapfile -t accel_subdirs <<< "$accel_subdirs" + #eval "accel_subdirs=(${accel_subdirs})" echo "accel_subdirs 2: ${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" From d030bc8cce3e227612c1d1481fc6b447187ecba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Apr 2026 00:02:47 +0200 Subject: [PATCH 19/42] use %q --- bot/build.sh | 2 +- create_tarball.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 750f1966..8bc4cf0c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$(printf '%s\n' \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\")" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$(printf '%q ' \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\")" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index 745bcb04..5f8dc8aa 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -68,8 +68,8 @@ echo "accel_subdirs 1: ${accel_subdirs}" if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... #read -r -a accel_subdirs <<< ${accel_subdirs} - mapfile -t accel_subdirs <<< "$accel_subdirs" - #eval "accel_subdirs=(${accel_subdirs})" + #mapfile -t accel_subdirs <<< "$accel_subdirs" + eval "accel_subdirs=(${accel_subdirs})" echo "accel_subdirs 2: ${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" From e18733a55da5dfba6cb79de79fa404bd0d612438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Apr 2026 00:08:21 +0200 Subject: [PATCH 20/42] remove quotes --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 8bc4cf0c..e6879e86 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$(printf '%q ' \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\")" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(printf '%q ' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From 706943b3cb83713ebfdd9a7ba3e3230d6c9bffbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 09:39:23 +0200 Subject: [PATCH 21/42] pass accels as '+'-separated string to create_tarball.sh --- bot/build.sh | 2 +- create_tarball.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index e6879e86..a053cb17 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -332,6 +332,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(printf '%q ' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(printf '%s+' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index 5f8dc8aa..f632ca4c 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -69,7 +69,8 @@ if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... #read -r -a accel_subdirs <<< ${accel_subdirs} #mapfile -t accel_subdirs <<< "$accel_subdirs" - eval "accel_subdirs=(${accel_subdirs})" + #eval "accel_subdirs=(${accel_subdirs})" + IFS='+' read -ra accel_subdirs <<< "${accel_subdirs}" echo "accel_subdirs 2: ${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" From 2a1264f5093c33207a1922b1a5b73c2425578ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 10:29:40 +0200 Subject: [PATCH 22/42] try to loop over prefixes in tarball --- bot/check-build.sh | 116 ++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index b3bd798c..5ef14522 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -500,68 +500,76 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then repo_version=$(cfg_get_value "repository" "repo_version") os_type=$(cfg_get_value "architecture" "os_type") software_subdir=$(cfg_get_value "architecture" "software_subdir") - accelerator=$(cfg_get_value "architecture" "accelerator") - prefix="${repo_version}/software/${os_type}/${software_subdir}" + accelerators=$(cfg_get_value "architecture" "accelerator") + cpu_prefix="${repo_version}/software/${os_type}/${software_subdir}" + prefixes=( "${cpu_prefix" ) # if we build for an accelerator, the prefix is different - if [[ ! -z ${accelerator} ]]; then - prefix="${prefix}/accel/${accelerator}" + if [[ ! -z ${accelerators} ]]; then + IFS='+' read -ra accelerators <<< "$accelerators" + # prepend the cpu prefix and "accel" to the accelerator names + prefixes="${accelerators[@]/#/${cpu_prefix}/accel/}" + #prefixes=( "${prefixes[@]}" "${accelerator_prefixes[@]}" ) fi - # extract directories/entries from tarball content - modules_entries=$(grep "${prefix}/modules" ${tmpfile}) - software_entries=$(grep "${prefix}/software" ${tmpfile}) - reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) - reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") - other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) - modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) - software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) - artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" comment_artifacts_list="" comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${modules} ]]; then
-            while IFS= read -r mod ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
-            done <<< "${modules}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${software_pkgs} ]]; then
-            while IFS= read -r sw_pkg ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
-            done <<< "${software_pkgs}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${reprod_shortened} ]]; then
-            while IFS= read -r reprod ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${reprod})"
-            done <<< "${reprod_shortened}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${other_shortened} ]]; then
-            while IFS= read -r other ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
-            done <<< "${other_shortened}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" + + for prefix in "${prefixes[@]}"; do + + # extract directories/entries from tarball content + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) + reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${modules} ]]; then
+                while IFS= read -r mod ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
+                done <<< "${modules}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${software_pkgs} ]]; then
+                while IFS= read -r sw_pkg ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
+                done <<< "${software_pkgs}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${reprod_shortened} ]]; then
+                while IFS= read -r reprod ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${reprod})"
+                done <<< "${reprod_shortened}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${other_shortened} ]]; then
+                while IFS= read -r other ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
+                done <<< "${other_shortened}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + done else comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" fi From 9ba104f5132f79f249f0293466aa4e5471a4b68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 10:42:20 +0200 Subject: [PATCH 23/42] debug --- bot/check-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index 5ef14522..f44b23d7 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -511,6 +511,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then prefixes="${accelerators[@]/#/${cpu_prefix}/accel/}" #prefixes=( "${prefixes[@]}" "${accelerator_prefixes[@]}" ) fi + echo "${prefixes[@]}" artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" comment_artifacts_list="" @@ -518,6 +519,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" for prefix in "${prefixes[@]}"; do + echo "check-build prefix: ${prefix}" # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) From adc821b04be7653feacd839cc647219ffb3f940d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 10:51:50 +0200 Subject: [PATCH 24/42] add missing } --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index f44b23d7..d29cd0f4 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -502,7 +502,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then software_subdir=$(cfg_get_value "architecture" "software_subdir") accelerators=$(cfg_get_value "architecture" "accelerator") cpu_prefix="${repo_version}/software/${os_type}/${software_subdir}" - prefixes=( "${cpu_prefix" ) + prefixes=( "${cpu_prefix}" ) # if we build for an accelerator, the prefix is different if [[ ! -z ${accelerators} ]]; then From e56b29225de24f2abbf7db809c6f19dd2acd0667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:01:05 +0200 Subject: [PATCH 25/42] make sure that prefixes is an array --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index d29cd0f4..88510c0a 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -508,7 +508,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then if [[ ! -z ${accelerators} ]]; then IFS='+' read -ra accelerators <<< "$accelerators" # prepend the cpu prefix and "accel" to the accelerator names - prefixes="${accelerators[@]/#/${cpu_prefix}/accel/}" + prefixes=( "${accelerators[@]/#/${cpu_prefix}/accel/}" ) #prefixes=( "${prefixes[@]}" "${accelerator_prefixes[@]}" ) fi echo "${prefixes[@]}" From e1f2e1c9bfcc9becd45ddca662165b5ed47643ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:15:17 +0200 Subject: [PATCH 26/42] for 'other files', first filter the ones with the current prefix --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 88510c0a..d49d4cfb 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -526,7 +526,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then software_entries=$(grep "${prefix}/software" ${tmpfile}) reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") + other_entries=$(cat ${tmpfile} | grep "${prefix}" | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) From e17fbaa7d30f93495385b8cb8567d989759cb209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:24:50 +0200 Subject: [PATCH 27/42] remove echo statements --- bot/check-build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index d49d4cfb..4bdc58e8 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -511,7 +511,6 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then prefixes=( "${accelerators[@]/#/${cpu_prefix}/accel/}" ) #prefixes=( "${prefixes[@]}" "${accelerator_prefixes[@]}" ) fi - echo "${prefixes[@]}" artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" comment_artifacts_list="" @@ -519,8 +518,6 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" for prefix in "${prefixes[@]}"; do - echo "check-build prefix: ${prefix}" - # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) From 3a36d3692b5850f0dec8b2f4fadc5de79674cc68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:26:34 +0200 Subject: [PATCH 28/42] remove commented lines --- bot/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a053cb17..f7e02a1d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -256,9 +256,6 @@ IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" EESSI_ACCELERATOR_TARGET_OVERRIDES=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") if [[ -n "$ACCEL_OVERRIDES" ]]; then for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do - # bot job config does not include accel subdirectory - #export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/${ACCEL_OVERRIDE}" - #echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/${ACCEL_OVERRIDE}'" export EESSI_ACCELERATOR_TARGET_OVERRIDE="${ACCEL_OVERRIDE}" echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${ACCEL_OVERRIDE}'" echo "Executing command to build software:" From 18cb0c76799187c67b5ac7bd7c9de97858ca51f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:27:25 +0200 Subject: [PATCH 29/42] remove commented lines and echo statements --- create_tarball.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index f632ca4c..2472ac94 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,22 +64,14 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} -echo "accel_subdirs 1: ${accel_subdirs}" if [ -n "${accel_subdirs}" ]; then # convert accel_subdirs into an array... - #read -r -a accel_subdirs <<< ${accel_subdirs} - #mapfile -t accel_subdirs <<< "$accel_subdirs" - #eval "accel_subdirs=(${accel_subdirs})" IFS='+' read -ra accel_subdirs <<< "${accel_subdirs}" - echo "accel_subdirs 2: ${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" - echo "accel_subdirs 3: ${accel_subdirs}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" fi for subdir in ${sw_subdirs}; do - - echo "subdir: ${subdir}" if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${subdir}/modules -type f \! -name '.wh.*' >> ${files_list} From a91eeb885715802856e4c381e5a2964d220a9898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 11:42:20 +0200 Subject: [PATCH 30/42] clean up code --- bot/build.sh | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index f7e02a1d..7b5e56c3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -251,33 +251,27 @@ build_outerr=$(mktemp build.outerr.XXXX) # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") -IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" -# prepend accel/ to all array elements -EESSI_ACCELERATOR_TARGET_OVERRIDES=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") -if [[ -n "$ACCEL_OVERRIDES" ]]; then - for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do - export EESSI_ACCELERATOR_TARGET_OVERRIDE="${ACCEL_OVERRIDE}" - echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${ACCEL_OVERRIDE}'" - echo "Executing command to build software:" - echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" - echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" - $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} - - # determine temporary directory to resume from for the next accelerator, - # as we want to combine all accelerator builds into a single tarball in the end - BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) - BUILD_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") - done +if [[ -z ${ACCEL_OVERRIDES} ]]; then + EESSI_ACCELERATOR_TARGET_OVERRIDES=("") else - export EESSI_ACCELERATOR_TARGET_OVERRIDE="" + IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" + # prepend accel/ to all array elements + EESSI_ACCELERATOR_TARGET_OVERRIDES=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") +fi +for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do + export EESSI_ACCELERATOR_TARGET_OVERRIDE="${ACCEL_OVERRIDE}" + echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${ACCEL_OVERRIDE}'" echo "Executing command to build software:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} -fi -echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'" + + # determine temporary directory to resume from for the next accelerator, + # as we want to combine all accelerator builds into a single tarball in the end + BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) + BUILD_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") +done # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step From f764500c58bda5108a9a377e0aa73be95d1ad57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 12:11:14 +0200 Subject: [PATCH 31/42] try to encode list of accelerators in tarball filename --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 7b5e56c3..5c1efb4f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -306,8 +306,8 @@ source $software_layer_dir/init/eessi_defaults if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - #export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} accel ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + accels=$(printf '%s-' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]//\//-}") + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${accels} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball From 11bf64989a557be9538ce229f51f5cd8cdfa2647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 12:26:33 +0200 Subject: [PATCH 32/42] add comment --- bot/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 5c1efb4f..a60150d8 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -306,8 +306,9 @@ source $software_layer_dir/init/eessi_defaults if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - accels=$(printf '%s-' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]//\//-}") - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${accels} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + # replace slashes in accelerator names by a hyphen, and concatenate them into a hypen-separated string + filename_accelerators=$(printf '%s-' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]//\//-}") + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball From 2f0f65191ef0556c4047280c4f5084cc58c6bb43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 12:29:08 +0200 Subject: [PATCH 33/42] remove accel easystack --- .../2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml diff --git a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml deleted file mode 100644 index 66a4a1f9..00000000 --- a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - pmt-1.3.1-GCCcore-13.3.0-CUDA-12.6.0.eb From 1d95c86d628b65e6b74da16ee9985e83f00a351a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 12:29:13 +0200 Subject: [PATCH 34/42] add easystack --- .../software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml diff --git a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml new file mode 100644 index 00000000..44fe4d92 --- /dev/null +++ b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - cowsay-3.04.eb From 2a63e161286c8930c5e2b5eb2c9682ab02c2f80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 12:47:18 +0200 Subject: [PATCH 35/42] debug --- create_tarball.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 2472ac94..1f16ae77 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -65,11 +65,14 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} if [ -n "${accel_subdirs}" ]; then + echo "accel subdirs: ${accel_subdirs}" # convert accel_subdirs into an array... IFS='+' read -ra accel_subdirs <<< "${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" + echo "accel subdirs: ${accel_subdirs}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" + echo "sw subdirs: ${sw_subdirs}" fi for subdir in ${sw_subdirs}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then From 72e8fab3c0f5b10d00cc66e4e1f20e12665c4712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:00:34 +0200 Subject: [PATCH 36/42] use IFS instead of printf for concatenating array items --- bot/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a60150d8..c5f6377a 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -307,8 +307,8 @@ if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else # replace slashes in accelerator names by a hyphen, and concatenate them into a hypen-separated string - filename_accelerators=$(printf '%s-' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]//\//-}") - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + filename_accelerators=$(IFS=-; echo "$EESSI_ACCELERATOR_TARGET_OVERRIDES[*]//\//-}") + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball @@ -324,6 +324,6 @@ echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(printf '%s+' "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(IFS=+; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From ce9af9477703f06d687f60eb2265d677eebe76f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:00:46 +0200 Subject: [PATCH 37/42] clean up --- bot/check-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 4bdc58e8..9ea14f96 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -502,14 +502,13 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then software_subdir=$(cfg_get_value "architecture" "software_subdir") accelerators=$(cfg_get_value "architecture" "accelerator") cpu_prefix="${repo_version}/software/${os_type}/${software_subdir}" - prefixes=( "${cpu_prefix}" ) + prefixes=("${cpu_prefix}") # if we build for an accelerator, the prefix is different if [[ ! -z ${accelerators} ]]; then IFS='+' read -ra accelerators <<< "$accelerators" # prepend the cpu prefix and "accel" to the accelerator names - prefixes=( "${accelerators[@]/#/${cpu_prefix}/accel/}" ) - #prefixes=( "${prefixes[@]}" "${accelerator_prefixes[@]}" ) + prefixes=("${accelerators[@]/#/${cpu_prefix}/accel/}") fi artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" From 22030ff3cbc0e62887cf9f33cf9a46579c651a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:41:06 +0200 Subject: [PATCH 38/42] improve passing of accelerators to create_tarball --- bot/build.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index c5f6377a..7eb0c773 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -303,11 +303,11 @@ source $software_layer_dir/init/eessi_defaults # append the project (subdirectory) name to the end tarball name. This is information # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is # appended -if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then +if [[ -z ${ACCEL_OVERRIDES} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else # replace slashes in accelerator names by a hyphen, and concatenate them into a hypen-separated string - filename_accelerators=$(IFS=-; echo "$EESSI_ACCELERATOR_TARGET_OVERRIDES[*]//\//-}") + filename_accelerators=$(IFS=-; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]//\//-}") export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi @@ -320,10 +320,11 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} # TODO should we make this a configurable parameter of eessi_container.sh using # /tmp as default? TMP_IN_CONTAINER=/tmp +tarball_accelerators=$(IFS=+; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}") echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"$tarball_accelerators\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} $(IFS=+; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}") /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$tarball_accelerators" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From b59cd42c2aefc684ea04658733dc0f96b47c762e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:50:20 +0200 Subject: [PATCH 39/42] remove echo commands --- create_tarball.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 1f16ae77..2472ac94 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -65,14 +65,11 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} if [ -n "${accel_subdirs}" ]; then - echo "accel subdirs: ${accel_subdirs}" # convert accel_subdirs into an array... IFS='+' read -ra accel_subdirs <<< "${accel_subdirs}" # and prepend the cpu_arch_subdir to all elements accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" - echo "accel subdirs: ${accel_subdirs}" sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" - echo "sw subdirs: ${sw_subdirs}" fi for subdir in ${sw_subdirs}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then From a9bc235622ee3291775670a43db0f2de5895f520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:51:25 +0200 Subject: [PATCH 40/42] remove easystack --- .../software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml diff --git a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml deleted file mode 100644 index 44fe4d92..00000000 --- a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.3.0-system.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - cowsay-3.04.eb From 6ffa25ab3761bc6a8a169cb15a0c3369440ac127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 14:52:18 +0200 Subject: [PATCH 41/42] add accel easystack --- .../2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml diff --git a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml new file mode 100644 index 00000000..66a4a1f9 --- /dev/null +++ b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml @@ -0,0 +1,2 @@ +easyconfigs: + - pmt-1.3.1-GCCcore-13.3.0-CUDA-12.6.0.eb From cfe9667195c3a04382aff54b2328b951fc723c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 May 2026 15:06:52 +0200 Subject: [PATCH 42/42] remove easystack --- .../2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml diff --git a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml b/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml deleted file mode 100644 index 66a4a1f9..00000000 --- a/easystacks/software.eessi.io/2025.06/accel/nvidia/eessi-2025.06-eb-5.3.0.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - pmt-1.3.1-GCCcore-13.3.0-CUDA-12.6.0.eb