From 324f5363d490621d131649c124b0d549aec78228 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:02:44 +0200 Subject: [PATCH 1/9] Try to bind repo from the host, see if we can reproduce our original issue --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 290444f1..7295ad38 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -184,7 +184,7 @@ COMMON_ARGS+=("--mode" "run") [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -[[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID}") +[[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID},mode=bind") # Also expose software.eessi.io when configured for dev.eessi.io # Need software.eessi.io for the compat layer From ff240973a9f2f292e398e769c6db231e896d176a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:04:58 +0200 Subject: [PATCH 2/9] Add test easystacks with which we have seen the original issue --- ...-5.1.1-rebuild-2023a-for-cuda-sanity-check.yml | 15 +++++++++++++++ ...-5.1.1-rebuild-2023b-for-cuda-sanity-check.yml | 14 ++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023a-for-cuda-sanity-check.yml create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023b-for-cuda-sanity-check.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023a-for-cuda-sanity-check.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023a-for-cuda-sanity-check.yml new file mode 100644 index 00000000..5d701e9e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023a-for-cuda-sanity-check.yml @@ -0,0 +1,15 @@ +# We'll rebuild all CUDA software, for various reasons +# 1. We now have a proper CUDA sanity check, and if anything was 'wrong' with our current CUDA installs, we'd like +# to know about it +# 2. The PR implementing a CI to check for differences between officially supported CUDA Compute Capabilities shows +# that there are a lot of missing installations https://github.com/EESSI/software-layer/pull/1087 . A rebuild PR like +# this will have the convenient side effect of filling all those holes +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA + - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb + - LightGBM-4.5.0-foss-2023a-CUDA-12.1.1.eb + diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023b-for-cuda-sanity-check.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023b-for-cuda-sanity-check.yml new file mode 100644 index 00000000..c9f1f9b4 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250805-eb-5.1.1-rebuild-2023b-for-cuda-sanity-check.yml @@ -0,0 +1,14 @@ +# We'll rebuild all CUDA software, for various reasons +# 1. We now have a proper CUDA sanity check, and if anything was 'wrong' with our current CUDA installs, we'd like +# to know about it +# 2. The PR implementing a CI to check for differences between officially supported CUDA Compute Capabilities shows +# that there are a lot of missing installations https://github.com/EESSI/software-layer/pull/1087 . A rebuild PR like +# this will have the convenient side effect of filling all those holes +easyconfigs: + - CUDA-12.4.0.eb: + options: + accept-eula-for: CUDA + - UCX-CUDA-1.15.0-GCCcore-13.2.0-CUDA-12.4.0.eb + - UCC-CUDA-1.2.0-GCCcore-13.2.0-CUDA-12.4.0.eb + - OSU-Micro-Benchmarks-7.5-gompi-2023b-CUDA-12.4.0.eb + - GROMACS-2024.4-foss-2023b-CUDA-12.4.0.eb From e3044260b58f0dfef7b582b2ab2ba73657963548 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:37:49 +0200 Subject: [PATCH 3/9] Fix mount option --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 7295ad38..6854aa10 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -184,7 +184,7 @@ COMMON_ARGS+=("--mode" "run") [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -[[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID},mode=bind") +[[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID},mount=bind") # Also expose software.eessi.io when configured for dev.eessi.io # Need software.eessi.io for the compat layer From 99fdf09f195d90efa8f5bb94726d8d2b4e6c5174 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:40:55 +0200 Subject: [PATCH 4/9] Debugging... --- install_software_layer.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_software_layer.sh b/install_software_layer.sh index 8b88e757..2c9243b7 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,6 @@ #!/bin/bash +echo "TRYING TO LS FIRST" +ls -al /cvmfs/software.eessi.io base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults $base_dir/run_in_compat_layer_env.sh $base_dir/EESSI-install-software.sh "$@" From 4b686e97ba32bbed6e95601ded5cced158b53340 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:43:35 +0200 Subject: [PATCH 5/9] More debugging --- install_software_layer.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_software_layer.sh b/install_software_layer.sh index 2c9243b7..04b026c9 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,6 +1,8 @@ #!/bin/bash echo "TRYING TO LS FIRST" ls -al /cvmfs/software.eessi.io +echo "TRYING TO WRITE AT THIS LEVEL" +touch /cvmfs/software.eessi.io/foo base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults $base_dir/run_in_compat_layer_env.sh $base_dir/EESSI-install-software.sh "$@" From 674fe2b9922f117e7b0b9a432523f0c66d607e82 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:45:09 +0200 Subject: [PATCH 6/9] More debugging --- install_software_layer.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install_software_layer.sh b/install_software_layer.sh index 04b026c9..0b4251c4 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -3,6 +3,8 @@ echo "TRYING TO LS FIRST" ls -al /cvmfs/software.eessi.io echo "TRYING TO WRITE AT THIS LEVEL" touch /cvmfs/software.eessi.io/foo +echo "SEE IF WE HAVE A FOO FILE" +ls -al /cvmfs/software.eessi.io base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults $base_dir/run_in_compat_layer_env.sh $base_dir/EESSI-install-software.sh "$@" From 87e89d8695efc3a6f1655de0ff7b155084722c66 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 12:49:04 +0200 Subject: [PATCH 7/9] More debugging --- EESSI-install-software.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index bf5c59ca..fe060c28 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -188,6 +188,13 @@ pr_diff=$(ls [0-9]*.diff | head -1) export PR_DIFF="$PWD/$pr_diff" # Only run install_scripts.sh if not in dev.eessi.io for security +echo "EESSI-software-layer.sh: TRYING TO LS" +ls -al /cvmfs/software.eessi.io +echo "EESSI-software-layer.sh: TRYING TO TOUCH" +touch /cvmfs/software.eessi.io/bar +echo "EESSI-software-layer.sh CHECK FOR NEW FILE" +ls -al /cvmfs/software.eessi.io + if [[ -z ${EESSI_DEV_PROJECT} ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION} --eessi-version ${EESSI_VERSION} fi From fdc15789ca2ef5b01e86dab897af6dda0756eafd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 13:01:37 +0200 Subject: [PATCH 8/9] Foo from install scripts --- install_scripts.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/install_scripts.sh b/install_scripts.sh index e6220c2f..e142836c 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -231,6 +231,13 @@ copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" # note: the commands below are always run, regardless of whether the scripts were changed, # but that should be fine (no changes are made if version placeholder is not present anymore) +echo "install-scripts.sh: TRYING TO LS" +ls -al /cvmfs/software.eessi.io +echo "install-scripts.sh: TRYING TO TOUCH" +touch /cvmfs/software.eessi.io/foo_install_scripts +echo "install-scripts.sh: CHECK FOR NEW FILE" +ls -al /cvmfs/software.eessi.io + # make sure that scripts in init/ and scripts/ use correct EESSI version sed_update_if_changed "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/eessi_defaults From 9654c0b87212a7127bbe4d52a3a19773b648a048 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Aug 2025 13:40:41 +0200 Subject: [PATCH 9/9] More debugging --- install_scripts.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/install_scripts.sh b/install_scripts.sh index e142836c..818e9f2d 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -46,6 +46,16 @@ sed_update_if_changed() { local sed_command="$1" local file="$2" + + echo "install-scripts.sh:sed_update_if_changed: TRYING TO LS" + ls -al /cvmfs/software.eessi.io/versions/2023.06/init + echo "install-scripts.sh:sed_update_if_changed: TRYING TO TOUCH" + touch /cvmfs/software.eessi.io/versions/2023.06/init/foo_install_scripts_sed_update + mktemp foo.XXXXXXX + echo "install-scripts.sh:sed_update_if_changed: CHECK FOR NEW FILE" + ls -al /cvmfs/software.eessi.io/versions/2023.06/init + + local tmp_file="$(mktemp "${file}.XXXXXX")" sed "$sed_command" "$file" > "$tmp_file" || {