diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 7a8f5fa4..2875cb2e 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -45,13 +45,14 @@ jobs: - name: test load_easybuild_module.sh script run: | + export SINGULARITY_CACHEDIR=$PWD # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, # since Python in compat layer is Python 3.11.x; # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version - for EB_VERSION in '4.6.0'; do + for EB_VERSION in '5.1.0'; do # Create script that uses load_easybuild_module.sh which we can run in compat layer environment # note: Be careful with single vs double quotes below! # ${EB_VERSION} should be expanded, so use double quotes; @@ -91,6 +92,7 @@ jobs: - name: test install_software_layer.sh script run: | + export SINGULARITY_CACHEDIR=$PWD # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" # force using x86_64/generic, to avoid triggering an installation from scratch @@ -105,6 +107,7 @@ jobs: - name: test create_directory_tarballs.sh script run: | + export SINGULARITY_CACHEDIR=$PWD # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" @@ -113,12 +116,13 @@ jobs: # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container - ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh "${{matrix.EESSI_VERSION}}" # check if tarballs have been produced ls -l *.tar.gz - name: test create_lmodsitepackage.py script run: | + export SINGULARITY_CACHEDIR=$PWD # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" @@ -140,3 +144,22 @@ jobs: for pattern in "^Site Pkg location.*/software-layer-scripts/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer-scripts/.lmod/SitePackage.lua"; do grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) done + + - name: Mount EESSI CernVM-FS repository + uses: eessi/github-action-eessi@v3 + with: + eessi_stack_version: ${{matrix.EESSI_VERSION}} + use_eessi_module: true + + - name: Verify that mounted repositories are passed through directly + run: | + export SINGULARITY_CACHEDIR=$PWD + # run wrapper script + capture & check output + export SINGULARITY_BIND="${PWD}:/software-layer-scripts" + # make sure that correct EESSI version is used (required because default is a placeholder version) + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" + + out="${PWD}/eb-${EB_VERSION}.out" + ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh ls 2>&1 | tee ${out} + echo $(grep "SINGULARITY_BIND" ${out}) + grep "SINGULARITY_BIND" ${out} | grep "software.eessi.io" || (echo "software.eessi.io did not seem to be bind mounted!" && exit 1) diff --git a/eessi_container.sh b/eessi_container.sh index 00a03227..c0be36fc 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -747,7 +747,14 @@ declare -a EESSI_FUSE_MOUNTS=() # mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) unless env var # EESSI_DO_NOT_MOUNT_CVMFS_CONFIG_CERN_CH is defined if [ -z ${EESSI_DO_NOT_MOUNT_CVMFS_CONFIG_CERN_CH+x} ]; then - EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") + if cvmfs_config stat cvmfs-config.cern.ch ; then + # if the system has the repo available just pass it through + BIND_PATHS="${BIND_PATHS},/cvmfs/cvmfs-config.cern.ch:/cvmfs/cvmfs-config.cern.ch" + else + # otherwise use apptainer capabilities to make it available within the container + EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") + export EESSI_FUSE_MOUNTS + fi fi @@ -795,9 +802,14 @@ do echo " session. Will use it as left-most directory in 'lowerdir' argument for fuse-overlayfs." # make the target CernVM-FS repository available under /cvmfs_ro - export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + if cvmfs_config stat ${cvmfs_repo_name} ; then + # if the system has the repo available just pass it through + BIND_PATHS="${BIND_PATHS},/cvmfs/${cvmfs_repo_name}:/cvmfs_ro/${cvmfs_repo_name}" + else + # otherwise use apptainer capabilities to make it available within the container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + fi # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then @@ -827,10 +839,15 @@ do # basic "ro" access that doesn't require any fuseoverlay-fs echo "Mounting '${cvmfs_repo_name}' 'read-only' without fuse-overlayfs." - export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS + if cvmfs_config stat ${cvmfs_repo_name} ; then + # if the system has the repo available just pass it through + BIND_PATHS="${BIND_PATHS},/cvmfs/${cvmfs_repo_name}:/cvmfs/${cvmfs_repo_name}" + else + # otherwise use apptainer capabilities to make it available within the container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + fi fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories @@ -839,10 +856,15 @@ do [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" - # set environment variables for fuse mounts in Singularity container - export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + # make the target CernVM-FS repository available under /cvmfs_ro + if cvmfs_config stat ${cvmfs_repo_name} ; then + # if the system has the repo available just pass it through + BIND_PATHS="${BIND_PATHS},/cvmfs/${cvmfs_repo_name}:/cvmfs_ro/${cvmfs_repo_name}" + else + # otherwise use apptainer capabilities to make it available within the container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + fi if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"