Support multiple EESSI versions by using ReFrame environments

config/github_actions.py

@@ -16,7 +16,6 @@
                     'name': 'default',
                     'scheduler': 'local',
                     'launcher': 'local',
-                    'environs': ['default'],
                     'features': [FEATURES.CPU] + list(SCALES.keys()),
                     'processor': {
                         'num_cpus': 2,

eessi/testsuite/common_config.py

@@ -42,8 +42,11 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True
     :param site_configuration: site configuration dictionary
     :param set_memory: whether to set memory resources
     """
-    environments = [{'name': 'default'}]
-    environs = ['default']
+    environments = [
+        {'name': 'EESSI-2023.06', 'modules': ['EESSI/2023.06']},
+        {'name': 'EESSI-2025.06', 'modules': ['EESSI/2025.06']},
+    ]
+    environs = ['EESSI-2023.06', 'EESSI-2025.06']
     use_nodes_option = True
     if set_memory:
         resources_memory = [{
@@ -58,16 +61,23 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True
     }]
 
     if 'environments' in site_configuration and site_configuration['environments'] != environments:
-        getlogger().info(f"Changing environments in site config to {environments}")
-    site_configuration['environments'] = environments
+        msg = f"Appending environments {environments} to the environments already present in the site_configuration"
+        msg += f" ({site_configuration['environments']})"
+        getlogger().info(msg)
+        site_configuration['environments'].extend(environments)
+    else:
+        site_configuration['environments'] = environments
 
     for system in site_configuration.get('systems', []):
         for partition in system.get('partitions', []):
             # Set or overwrite the partition environment
             if 'environs' in partition and partition['environs'] != environs:
-                getlogger().info(
-                    f"Changing environs in site config to {environs} for {system['name']}:{partition['name']}")
-            partition['environs'] = environs
+                msg = f"Appending environs {environs} to the existing environs ({partition['environs']})"
+                msg += f" for {system['name']}:{partition['name']}"
+                getlogger().info(msg)
+                partition['environs'].extend(environs)
+            else:
+                partition['environs'] = environs
 
             # Set or overwrite the 'use_nodes_option' scheduler option, if this is a SLURM-like scheduler
             if partition['scheduler'] in ['slurm', 'squeue']:

eessi/testsuite/eessi_mixin.py

@@ -9,6 +9,8 @@
     from reframe.core.pipeline import RegressionMixin as RegressionTestPlugin
 from reframe.utility.sanity import make_performance_function
 import reframe.utility.sanity as sn
+from reframe.core.runtime import valid_sysenv_comb
+from reframe import VERSION as reframe_version
 
 from eessi.testsuite import check_process_binding, hooks
 from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES, TAGS
@@ -36,7 +38,7 @@ class EESSI_Mixin(RegressionTestPlugin):
     That definition needs to be done 'on time', i.e. early enough in the execution of the ReFrame pipeline.
     Here, we list which class attributes must be defined by the child class, and by (the end of) what phase:
 
-    - Init phase: device_type, scale, module_name, bench_name
+    - Init phase: device_type, scale, module_info, bench_name
     - Setup phase: compute_unit, required_mem_per_node
 
     The child class may also overwrite the following attributes:
@@ -80,13 +82,13 @@ class EESSI_Mixin(RegressionTestPlugin):
 
     # Note that the error for an empty parameter is a bit unclear for ReFrame 4.6.2, but that will hopefully improve
     # see https://github.com/reframe-hpc/reframe/issues/3254
-    # If that improves: uncomment the following to force the user to set module_name
-    # module_name = parameter()
+    # If that improves: uncomment the following to force the user to set module_info
+    # module_info = parameter()
 
     def __init_subclass__(cls, **kwargs):
         " set default values for built-in ReFrame attributes "
         super().__init_subclass__(**kwargs)
-        cls.valid_prog_environs = ['default']
+        cls.valid_prog_environs = ['*']
         cls.valid_systems = ['*']
         if not cls.time_limit:
             cls.time_limit = '1h'
@@ -137,7 +139,7 @@ def mark_all_files_readonly(self):
     def EESSI_mixin_validate_init(self):
         """Check that all variables that have to be set for subsequent hooks in the init phase have been set"""
         # List which variables we will need/use in the run_after('init') hooks
-        var_list = ['device_type', 'scale', 'module_name', 'measure_memory_usage']
+        var_list = ['device_type', 'scale', 'module_info', 'measure_memory_usage']
         for var in var_list:
             if not hasattr(self, var):
                 msg = "The variable '%s' should be defined in any test class that inherits" % var
@@ -149,7 +151,6 @@ def EESSI_mixin_validate_init(self):
         self.EESSI_mixin_validate_item_in_list('device_type', DEVICE_TYPES[:])
         self.EESSI_mixin_validate_item_in_list('scale', SCALES.keys())
         self.EESSI_mixin_validate_item_in_list('valid_systems', [['*']])
-        self.EESSI_mixin_validate_item_in_list('valid_prog_environs', [['default']])
 
     @run_after('init')
     def EESSI_mixin_run_after_init(self):
@@ -162,8 +163,6 @@ def EESSI_mixin_run_after_init(self):
         # Filter on which scales are supported by the partitions defined in the ReFrame configuration
         hooks.filter_supported_scales(self)
 
-        hooks.set_modules(self)
-
         if self.require_buildenv_module:
             hooks.add_buildenv_module(self)
 
@@ -174,7 +173,41 @@ def EESSI_mixin_run_after_init(self):
             err_msg = f"Invalid thread_binding value '{thread_binding}'. Valid values: 'true', 'compact', or 'false'."
             raise EESSIError(err_msg)
 
-        hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type)
+        # Unpack module_info
+        sys, env, mod = self.module_info
+        self.valid_prog_environs = [env]
+        self.module_name = [mod]
+
+        # Set modules
+        hooks.set_modules(self)
+
+        # Checks reframe version, and if newer or equal to 4.10, use the new functionality
+        # that allows combining sys:part notation with +feat.
+        syspart_feat_supported = False
+        try:
+            import semver
+            if semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0"):
+                syspart_feat_supported = True
+        except ImportError:
+            pass
+
+        # If we use reframe 4.10.0 or later, we can just set the valid system and the hook will
+        # append any relevant features. Otherwise, as a fallback, we set the features first
+        # (by calling the hook), then check if the sys:part combination from the find_modules triplet
+        # is in the list of valid combinations for the given features
+        if syspart_feat_supported:
+            self.valid_systems = [sys]
+            hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type)
+        else:
+            # Filter by defice type. E.g. add features based on whether CUDA appears in the module name
+            hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type)
+
+            # Check if partitions returned by find_modules satisfy the current features/extras in valid_systems
+            valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, env)]
+            if sys in valid_partitions:
+                self.valid_systems = [sys]
+            else:
+                self.valid_systems = []
 
         # Set scales as tags
         hooks.set_tag_scale(self)

eessi/testsuite/tests/apps/MetalWalls.py

@@ -46,7 +46,7 @@ class EESSI_MetalWalls_MW(MetalWallsCheck, EESSI_Mixin):
     # input files are downloaded
     readonly_files = ['']
 
-    module_name = parameter(find_modules('MetalWalls'))
+    module_info = parameter(find_modules('MetalWalls'))
     # For now, MetalWalls is being build for CPU targets only
     # compute_device = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU])
     device_type = parameter([DEVICE_TYPES.CPU])

eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py

@@ -15,7 +15,7 @@ class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest, EESSI_Mixin):
     nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large'])
     parallel_strategy = parameter([None, 'ddp'])
     # Both torchvision and PyTorch-bundle modules have everything needed to run this test
-    module_name = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle')))
+    module_info = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle')))
     executable = 'python'
     time_limit = '30m'
     readonly_files = ['get_free_socket.py', 'pytorch_synthetic_benchmark.py']

eessi/testsuite/tests/apps/QuantumESPRESSO.py

@@ -40,7 +40,7 @@
 @rfm.simple_test
 class EESSI_QuantumESPRESSO_PW(QEspressoPWCheck, EESSI_Mixin):
     time_limit = '30m'
-    module_name = parameter(find_modules('QuantumESPRESSO'))
+    module_info = parameter(find_modules('QuantumESPRESSO'))
     # For now, QE is built for CPU targets only
     device_type = parameter([DEVICE_TYPES.CPU])
     readonly_files = ['']

eessi/testsuite/tests/apps/cp2k/cp2k.py

@@ -17,7 +17,7 @@ class EESSI_CP2K(rfm.RunOnlyRegressionTest, EESSI_Mixin):
         ('QS/H2O-512', -8808.1439, 1e-4),
     ], fmt=lambda x: x[0], loggable=True)
 
-    module_name = parameter(find_modules('CP2K'))
+    module_info = parameter(find_modules('CP2K'))
     scale = parameter(SCALES.keys())
 
     executable = 'cp2k.popt'

eessi/testsuite/tests/apps/espresso/espresso.py

@@ -36,7 +36,7 @@ def filter_scales():
 
 
 class EESSI_ESPRESSO_base(rfm.RunOnlyRegressionTest):
-    module_name = parameter(find_modules('^ESPResSo$'))
+    module_info = parameter(find_modules('^ESPResSo$'))
     device_type = DEVICE_TYPES.CPU
     compute_unit = COMPUTE_UNITS.CPU
     time_limit = '300m'

eessi/testsuite/tests/apps/gromacs.py

@@ -49,7 +49,7 @@ def set_device_type(self):
 class EESSI_GROMACS(EESSI_GROMACS_base, EESSI_Mixin):
     scale = parameter(SCALES.keys())
     time_limit = '30m'
-    module_name = parameter(find_modules('GROMACS'))
+    module_info = parameter(find_modules('GROMACS'))
     # input files are downloaded
     readonly_files = ['']
     # executable_opts in addition to those set by the hpctestlib

eessi/testsuite/tests/apps/lammps/lammps.py

@@ -53,7 +53,7 @@ class EESSI_LAMMPS_base(rfm.RunOnlyRegressionTest):
     device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU])
 
     # Parameterize over all modules that start with LAMMPS
-    module_name = parameter(find_modules('LAMMPS'))
+    module_info = parameter(find_modules('LAMMPS'))
 
     all_readonly_files = True
     is_ci_test = True

eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py

@@ -48,7 +48,7 @@ class EESSI_lbmpy_pssrt(rfm.RunOnlyRegressionTest, EESSI_Mixin):
 
     launcher = 'local'  # no MPI module is loaded in this test
 
-    module_name = parameter(find_modules('lbmpy-pssrt'))
+    module_info = parameter(find_modules('lbmpy-pssrt'))
 
     readonly_files = ['mixing_layer_2D.py']
 

eessi/testsuite/tests/apps/lpc3d/lpc3d.py

@@ -44,7 +44,7 @@ class EESSI_LPC3D(rfm.RunOnlyRegressionTest, EESSI_Mixin):
 
     launcher = 'local'  # no MPI module is loaded in this test
 
-    module_name = parameter(find_modules('LPC3D'))
+    module_info = parameter(find_modules('LPC3D'))
 
     readonly_files = ['lattice_gas.inpt', 'pore_dens_freq_2neg.txt', 'psd.txt']
 

eessi/testsuite/tests/apps/numpy/numpy.py

@@ -24,7 +24,7 @@ class EESSI_NumPy(rfm.RunOnlyRegressionTest, EESSI_Mixin):
     executable = './np_ops.py'
     time_limit = '30m'
     readonly_files = ['np_ops.py']
-    module_name = parameter(find_modules('SciPy-bundle'))
+    module_info = parameter(find_modules('SciPy-bundle'))
     device_type = DEVICE_TYPES.CPU
     compute_unit = COMPUTE_UNITS.NODE
     scale = parameter([

eessi/testsuite/tests/apps/openfoam/openfoam.py

@@ -87,7 +87,7 @@ class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_64M(rfm.RunOnlyRegressionTest, EESSI_Mixi
     time_limit = '120m'
     readonly_files = ['']
     device_type = parameter([DEVICE_TYPES.CPU])
-    module_name = parameter(find_modules('OpenFOAM/v', name_only=False))
+    module_info = parameter(find_modules('OpenFOAM/v', name_only=False))
     valid_systems = ['*']
     scale = parameter(filter_scales_64M())
 

eessi/testsuite/tests/apps/osu.py

@@ -57,7 +57,7 @@ def filter_scales_coll():
 class EESSI_OSU_Base(osu_benchmark):
     """ base class for OSU tests """
     time_limit = '30m'
-    module_name = parameter(find_modules('OSU-Micro-Benchmarks'))
+    module_info = parameter(find_modules('OSU-Micro-Benchmarks'))
     used_cpus_per_task = 1
 
     # reset num_tasks_per_node from the hpctestlib: we handle it ourselves

eessi/testsuite/tests/apps/tensorflow/tensorflow.py

@@ -19,7 +19,7 @@
 class EESSI_TensorFlow(rfm.RunOnlyRegressionTest, EESSI_Mixin):
 
     # Parameterize over all modules that start with TensorFlow
-    module_name = parameter(find_modules('TensorFlow'))
+    module_info = parameter(find_modules('TensorFlow'))
 
     # Make CPU and GPU versions of this test
     device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU])

eessi/testsuite/tests/apps/walberla/walberla.py

@@ -44,7 +44,7 @@ class EESSI_WALBERLA_BACKWARD_FACING_STEP(rfm.RunOnlyRegressionTest, EESSI_Mixin
     time_limit = '30m'
     readonly_files = ['']
     device_type = parameter([DEVICE_TYPES.CPU])
-    module_name = parameter(find_modules('waLBerla'))
+    module_info = parameter(find_modules('waLBerla'))
     valid_systems = ['*']
     scale = parameter(filter_scales())
 

eessi/testsuite/utils.py

@@ -6,13 +6,14 @@
 import os
 import re
 import sys
-from typing import Iterator, List
+from typing import Iterator, List, Tuple
 
 import reframe as rfm
 from reframe.core.exceptions import ReframeFatalError
 from reframe.core.logging import getlogger
 import reframe.core.runtime as rt
 from reframe.frontend.printer import PrettyPrinter
+from reframe.utility import find_modules as rf_find_modules
 
 from eessi.testsuite.constants import DEVICE_TYPES
 
@@ -145,66 +146,13 @@ def get_avail_modules() -> List[str]:
     return _available_modules
 
 
-def find_modules(regex: str, name_only=True) -> Iterator[str]:
+def find_modules(substr, environ_mapping=None) -> Iterator[Tuple[str, str, str]]:
     """
-    Return all modules matching the regular expression regex. Note that since we use re.search,
-    a module matches if the regex matches the module name at any place. I.e. the match does
-    not have to be at the start of the smodule name
-
-    Arguments:
-    - regex: a regular expression
-    - name_only: regular expressions will only be matched on the module name, not the version (default: True).
-
-    Note: the name_only feature assumes anything after the last forward '/' is the version,
-    and strips that before doing a match.
-
-    Example
-
-    Suppose we have the following modules on a system:
-
-    gompic/2022a
-    gompi/2022a
-    CGAL/4.14.3-gompi-2022a
-
-    The following calls would return the following respective modules
-
-    find_modules('gompi') => [gompic/2022a, gompi/2022a]
-    find_modules('gompi$') => [gompi/2022a]
-    find_modules('gompi', name_only = False) => [gompic/2022a, gompi/2022a, CGAL/4.14.3-gompi-2022a]
-    find_modules('^gompi', name_only = False) => [gompic/2022a, gompi/2022a]
-    find_modules('^gompi/', name_only = False) => [gompi/2022a]
-    find_modules('-gompi-2022a', name_only = False) => [CGAL/4.14.3-gompi-2022a]
-
+    Wraps reframe.utility.find_modules in order to provide caching, so that we don't have to do repeated
+    module avail calls.
     """
-
-    if not isinstance(regex, str):
-        raise TypeError("'substr' argument must be a string")
-
-    seen = set()
-    dupes = []
-    for mod in get_avail_modules():
-        # The thing we yield should always be the original module name (orig_mod), including version
-        orig_mod = mod
-        if name_only:
-            # Remove trailing slashes from the regex (in case the callee forgot)
-            regex = regex.rstrip('/')
-            # Remove part after the last forward slash, as we assume this is the version
-            mod = re.sub('/[^/]*$', '', mod)
-        # Match the actual regular expression
-        log(f"Matching module {mod} with regex {regex}")
-        if re.search(regex, mod):
-            log("Match!")
-            if orig_mod in seen:
-                dupes.append(orig_mod)
-            else:
-                seen.add(orig_mod)
-            yield orig_mod
-
-    if dupes:
-        err_msg = "EESSI test-suite cannot handle duplicate modules. "
-        err_msg += "Please make sure that only one is available on your system. "
-        err_msg += f"The following modules have a duplicate on your system: {dupes}"
-        raise ValueError(err_msg)
+    # TODO: implement caching to make this function more efficient
+    return rf_find_modules(substr, environ_mapping)
 
 
 def get_tc_hierarchy(tcdict):