diff --git a/bot/build.sh b/bot/build.sh index 5c9f1af0..7eb0c773 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -161,16 +161,6 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" -# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} -ACCEL_OVERRIDE=$(cfg_get_value "architecture" "accelerator") -if [[ -n "$ACCEL_OVERRIDE" ]]; then - # bot job config does not include accel subdirectory - export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" -else - export EESSI_ACCELERATOR_TARGET_OVERRIDE="" -fi -echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'" - # Log the full lscpu and os-release info: lscpu > _bot_job${SLURM_JOB_ID}.lscpu cat /etc/os-release > _bot_job${SLURM_JOB_ID}.os @@ -259,11 +249,29 @@ fi # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) -echo "Executing command to build software:" -echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" -$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +ACCEL_OVERRIDES=$(cfg_get_value "architecture" "accelerator") +if [[ -z ${ACCEL_OVERRIDES} ]]; then + EESSI_ACCELERATOR_TARGET_OVERRIDES=("") +else + IFS='+' read -ra ACCEL_OVERRIDES_ARRAY <<< "$ACCEL_OVERRIDES" + # prepend accel/ to all array elements + EESSI_ACCELERATOR_TARGET_OVERRIDES=("${ACCEL_OVERRIDES_ARRAY[@]/#/accel/}") +fi +for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do + export EESSI_ACCELERATOR_TARGET_OVERRIDE="${ACCEL_OVERRIDE}" + echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${ACCEL_OVERRIDE}'" + echo "Executing command to build software:" + echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" + echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} + + # determine temporary directory to resume from for the next accelerator, + # as we want to combine all accelerator builds into a single tarball in the end + BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) + BUILD_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") +done # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step @@ -291,14 +299,16 @@ else fi # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults -# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we # append the project (subdirectory) name to the end tarball name. This is information # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is # appended -if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then +if [[ -z ${ACCEL_OVERRIDES} ]]; then export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + # replace slashes in accelerator names by a hyphen, and concatenate them into a hypen-separated string + filename_accelerators=$(IFS=-; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]//\//-}") + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${filename_accelerators} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball @@ -310,10 +320,11 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} # TODO should we make this a configurable parameter of eessi_container.sh using # /tmp as default? TMP_IN_CONTAINER=/tmp +tarball_accelerators=$(IFS=+; echo "${EESSI_ACCELERATOR_TARGET_OVERRIDES[*]}") echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"$tarball_accelerators\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$tarball_accelerators" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/bot/check-build.sh b/bot/check-build.sh index b3bd798c..9ea14f96 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -500,68 +500,74 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then repo_version=$(cfg_get_value "repository" "repo_version") os_type=$(cfg_get_value "architecture" "os_type") software_subdir=$(cfg_get_value "architecture" "software_subdir") - accelerator=$(cfg_get_value "architecture" "accelerator") - prefix="${repo_version}/software/${os_type}/${software_subdir}" + accelerators=$(cfg_get_value "architecture" "accelerator") + cpu_prefix="${repo_version}/software/${os_type}/${software_subdir}" + prefixes=("${cpu_prefix}") # if we build for an accelerator, the prefix is different - if [[ ! -z ${accelerator} ]]; then - prefix="${prefix}/accel/${accelerator}" + if [[ ! -z ${accelerators} ]]; then + IFS='+' read -ra accelerators <<< "$accelerators" + # prepend the cpu prefix and "accel" to the accelerator names + prefixes=("${accelerators[@]/#/${cpu_prefix}/accel/}") fi - # extract directories/entries from tarball content - modules_entries=$(grep "${prefix}/modules" ${tmpfile}) - software_entries=$(grep "${prefix}/software" ${tmpfile}) - reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) - reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") - other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) - modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) - software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) - artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" comment_artifacts_list="" comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${modules} ]]; then
-            while IFS= read -r mod ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
-            done <<< "${modules}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${software_pkgs} ]]; then
-            while IFS= read -r sw_pkg ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
-            done <<< "${software_pkgs}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${reprod_shortened} ]]; then
-            while IFS= read -r reprod ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${reprod})"
-            done <<< "${reprod_shortened}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" - comment_artifacts_list="${comment_artifacts_list}
"
-        if [[ ! -z ${other_shortened} ]]; then
-            while IFS= read -r other ; do
-                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
-            done <<< "${other_shortened}"
-        else
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
-        fi
-        comment_artifacts_list="${comment_artifacts_list}
" + + for prefix in "${prefixes[@]}"; do + # extract directories/entries from tarball content + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) + reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) + other_entries=$(cat ${tmpfile} | grep "${prefix}" | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${modules} ]]; then
+                while IFS= read -r mod ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
+                done <<< "${modules}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${software_pkgs} ]]; then
+                while IFS= read -r sw_pkg ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
+                done <<< "${software_pkgs}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${reprod_shortened} ]]; then
+                while IFS= read -r reprod ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${reprod})"
+                done <<< "${reprod_shortened}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" + comment_artifacts_list="${comment_artifacts_list}
"
+            if [[ ! -z ${other_shortened} ]]; then
+                while IFS= read -r other ; do
+                    comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
+                done <<< "${other_shortened}"
+            else
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
+            fi
+            comment_artifacts_list="${comment_artifacts_list}
" + done else comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" fi diff --git a/create_tarball.sh b/create_tarball.sh index d3251ba1..2472ac94 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -11,7 +11,7 @@ fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 -accel_subdir=$4 +accel_subdirs="$4" target_tarball=$5 tmpdir=`mktemp -d` @@ -64,11 +64,14 @@ fi # consider both CPU-only and accelerator subdirectories (if an accelerator was configured) sw_subdirs=${cpu_arch_subdir} -if [ -n "${accel_subdir}" ]; then - sw_subdirs="${sw_subdirs} ${cpu_arch_subdir}/${accel_subdir}" +if [ -n "${accel_subdirs}" ]; then + # convert accel_subdirs into an array... + IFS='+' read -ra accel_subdirs <<< "${accel_subdirs}" + # and prepend the cpu_arch_subdir to all elements + accel_subdirs="${accel_subdirs[@]/#/${cpu_arch_subdir}/}" + sw_subdirs="${sw_subdirs} ${accel_subdirs[@]}" fi for subdir in ${sw_subdirs}; do - if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${subdir}/modules -type f \! -name '.wh.*' >> ${files_list}