From 30d4393c06ab961475d006937cee36b865f1c1c6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 12 Mar 2026 16:43:45 +0100 Subject: [PATCH 01/12] Add initial functionality to schedule test for all EESSI environments in a single run, by configuring ReFrame environments and using ReFrame's own find_modules function --- eessi/testsuite/common_config.py | 7 ++++-- eessi/testsuite/eessi_mixin.py | 27 ++++++++++++++++++----- eessi/testsuite/tests/apps/gromacs.py | 5 +++-- eessi/testsuite/tests/apps/numpy/numpy.py | 4 ++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py index 44fbe0dd..71022668 100644 --- a/eessi/testsuite/common_config.py +++ b/eessi/testsuite/common_config.py @@ -42,8 +42,11 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True :param site_configuration: site configuration dictionary :param set_memory: whether to set memory resources """ - environments = [{'name': 'default'}] - environs = ['default'] + environments = [ + {'name': 'EESSI-2023.06', 'modules': ['EESSI/2023.06']}, + {'name': 'EESSI-2025.06', 'modules': ['EESSI/2025.06']}, + ] + environs = ['EESSI-2023.06', 'EESSI-2025.06'] use_nodes_option = True if set_memory: resources_memory = [{ diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 314ad45e..6cc33fa6 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -9,6 +9,7 @@ from reframe.core.pipeline import RegressionMixin as RegressionTestPlugin from reframe.utility.sanity import make_performance_function import reframe.utility.sanity as sn +from reframe.core.runtime import valid_sysenv_comb from eessi.testsuite import check_process_binding, hooks from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES, TAGS @@ -36,7 +37,7 @@ class EESSI_Mixin(RegressionTestPlugin): That definition needs to be done 'on time', i.e. early enough in the execution of the ReFrame pipeline. Here, we list which class attributes must be defined by the child class, and by (the end of) what phase: - - Init phase: device_type, scale, module_name, bench_name + - Init phase: device_type, scale, module_info, bench_name - Setup phase: compute_unit, required_mem_per_node The child class may also overwrite the following attributes: @@ -80,8 +81,8 @@ class EESSI_Mixin(RegressionTestPlugin): # Note that the error for an empty parameter is a bit unclear for ReFrame 4.6.2, but that will hopefully improve # see https://github.com/reframe-hpc/reframe/issues/3254 - # If that improves: uncomment the following to force the user to set module_name - # module_name = parameter() + # If that improves: uncomment the following to force the user to set module_info + # module_info = parameter() def __init_subclass__(cls, **kwargs): " set default values for built-in ReFrame attributes " @@ -137,7 +138,7 @@ def mark_all_files_readonly(self): def EESSI_mixin_validate_init(self): """Check that all variables that have to be set for subsequent hooks in the init phase have been set""" # List which variables we will need/use in the run_after('init') hooks - var_list = ['device_type', 'scale', 'module_name', 'measure_memory_usage'] + var_list = ['device_type', 'scale', 'module_info', 'measure_memory_usage'] for var in var_list: if not hasattr(self, var): msg = "The variable '%s' should be defined in any test class that inherits" % var @@ -162,8 +163,6 @@ def EESSI_mixin_run_after_init(self): # Filter on which scales are supported by the partitions defined in the ReFrame configuration hooks.filter_supported_scales(self) - hooks.set_modules(self) - if self.require_buildenv_module: hooks.add_buildenv_module(self) @@ -174,8 +173,24 @@ def EESSI_mixin_run_after_init(self): err_msg = f"Invalid thread_binding value '{thread_binding}'. Valid values: 'true', 'compact', or 'false'." raise EESSIError(err_msg) + # Unpack module_info + s, e, m = self.module_info + self.valid_prog_environs = [e] + self.module_name = [m] + + # Set modules + hooks.set_modules(self) + + # Filter by defice type. E.g. add features based on whether CUDA appears in the module name hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + # Check if the partitions returned by find_modules satisfy the current features/extras specified in valid_systems + valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] + if s in valid_partitions: + self.valid_systems = [s] + else: + self.valid_systems = [] + # Set scales as tags hooks.set_tag_scale(self) diff --git a/eessi/testsuite/tests/apps/gromacs.py b/eessi/testsuite/tests/apps/gromacs.py index ca64265b..4cdc7063 100644 --- a/eessi/testsuite/tests/apps/gromacs.py +++ b/eessi/testsuite/tests/apps/gromacs.py @@ -31,12 +31,13 @@ import reframe as rfm from reframe.core.builtins import parameter, run_after # added only to make the linter happy +from reframe.utility import find_modules from hpctestlib.sciapps.gromacs.benchmarks import gromacs_check from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES from eessi.testsuite.eessi_mixin import EESSI_Mixin -from eessi.testsuite.utils import find_modules, log +from eessi.testsuite.utils import log class EESSI_GROMACS_base(gromacs_check): @@ -49,7 +50,7 @@ def set_device_type(self): class EESSI_GROMACS(EESSI_GROMACS_base, EESSI_Mixin): scale = parameter(SCALES.keys()) time_limit = '30m' - module_name = parameter(find_modules('GROMACS')) + module_info = parameter(find_modules('GROMACS')) # input files are downloaded readonly_files = [''] # executable_opts in addition to those set by the hpctestlib diff --git a/eessi/testsuite/tests/apps/numpy/numpy.py b/eessi/testsuite/tests/apps/numpy/numpy.py index 8532fa80..1c58cea4 100644 --- a/eessi/testsuite/tests/apps/numpy/numpy.py +++ b/eessi/testsuite/tests/apps/numpy/numpy.py @@ -12,10 +12,10 @@ import reframe as rfm import reframe.utility.sanity as sn from reframe.core.builtins import parameter, run_after, run_before, sanity_function, variable +from reframe.utility import find_modules from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES from eessi.testsuite.eessi_mixin import EESSI_Mixin -from eessi.testsuite.utils import find_modules @rfm.simple_test @@ -24,7 +24,7 @@ class EESSI_NumPy(rfm.RunOnlyRegressionTest, EESSI_Mixin): executable = './np_ops.py' time_limit = '30m' readonly_files = ['np_ops.py'] - module_name = parameter(find_modules('SciPy-bundle')) + module_info = parameter(find_modules('SciPy-bundle')) device_type = DEVICE_TYPES.CPU compute_unit = COMPUTE_UNITS.NODE scale = parameter([ From ab0e7ce877f8ea3bfd6a8167dd4478edeb30edb6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 12 Mar 2026 17:00:50 +0100 Subject: [PATCH 02/12] Make all tests aware that find_modules now returns a module_info object instead --- eessi/testsuite/tests/apps/MetalWalls.py | 2 +- .../tests/apps/PyTorch/PyTorch_torchvision.py | 2 +- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 2 +- eessi/testsuite/tests/apps/cp2k/cp2k.py | 2 +- .../testsuite/tests/apps/espresso/espresso.py | 2 +- eessi/testsuite/tests/apps/gromacs.py | 3 +- eessi/testsuite/tests/apps/lammps/lammps.py | 2 +- .../tests/apps/lbmpy-pssrt/lbmpy-pssrt.py | 2 +- eessi/testsuite/tests/apps/lpc3d/lpc3d.py | 2 +- eessi/testsuite/tests/apps/numpy/numpy.py | 3 +- .../testsuite/tests/apps/openfoam/openfoam.py | 2 +- eessi/testsuite/tests/apps/osu.py | 2 +- .../tests/apps/tensorflow/tensorflow.py | 2 +- .../testsuite/tests/apps/walberla/walberla.py | 2 +- eessi/testsuite/utils.py | 64 ++----------------- 15 files changed, 19 insertions(+), 75 deletions(-) diff --git a/eessi/testsuite/tests/apps/MetalWalls.py b/eessi/testsuite/tests/apps/MetalWalls.py index a483e4a3..fc4c8cb5 100644 --- a/eessi/testsuite/tests/apps/MetalWalls.py +++ b/eessi/testsuite/tests/apps/MetalWalls.py @@ -51,7 +51,7 @@ class EESSI_MetalWalls_MW(MetalWallsCheck): # input files are downloaded readonly_files = [''] - module_name = parameter(find_modules('MetalWalls')) + module_info = parameter(find_modules('MetalWalls')) # For now, MetalWalls is being build for CPU targets only # compute_device = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) compute_device = parameter([DEVICE_TYPES.CPU]) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 32ae0994..7a80a565 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -15,7 +15,7 @@ class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest, EESSI_Mixin): nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large']) parallel_strategy = parameter([None, 'ddp']) # Both torchvision and PyTorch-bundle modules have everything needed to run this test - module_name = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle'))) + module_info = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle'))) executable = 'python' time_limit = '30m' readonly_files = ['get_free_socket.py', 'pytorch_synthetic_benchmark.py'] diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index eb51abef..52c3c710 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -40,7 +40,7 @@ @rfm.simple_test class EESSI_QuantumESPRESSO_PW(QEspressoPWCheck, EESSI_Mixin): time_limit = '30m' - module_name = parameter(find_modules('QuantumESPRESSO')) + module_info = parameter(find_modules('QuantumESPRESSO')) # For now, QE is built for CPU targets only device_type = parameter([DEVICE_TYPES.CPU]) readonly_files = [''] diff --git a/eessi/testsuite/tests/apps/cp2k/cp2k.py b/eessi/testsuite/tests/apps/cp2k/cp2k.py index 53de55d5..78e8605d 100644 --- a/eessi/testsuite/tests/apps/cp2k/cp2k.py +++ b/eessi/testsuite/tests/apps/cp2k/cp2k.py @@ -17,7 +17,7 @@ class EESSI_CP2K(rfm.RunOnlyRegressionTest, EESSI_Mixin): ('QS/H2O-512', -8808.1439, 1e-4), ], fmt=lambda x: x[0], loggable=True) - module_name = parameter(find_modules('CP2K')) + module_info = parameter(find_modules('CP2K')) scale = parameter(SCALES.keys()) executable = 'cp2k.popt' diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index d1e9a56c..d231b9ae 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -36,7 +36,7 @@ def filter_scales(): class EESSI_ESPRESSO_base(rfm.RunOnlyRegressionTest): - module_name = parameter(find_modules('^ESPResSo$')) + module_info = parameter(find_modules('^ESPResSo$')) device_type = DEVICE_TYPES.CPU compute_unit = COMPUTE_UNITS.CPU time_limit = '300m' diff --git a/eessi/testsuite/tests/apps/gromacs.py b/eessi/testsuite/tests/apps/gromacs.py index 4cdc7063..2a0630f8 100644 --- a/eessi/testsuite/tests/apps/gromacs.py +++ b/eessi/testsuite/tests/apps/gromacs.py @@ -31,13 +31,12 @@ import reframe as rfm from reframe.core.builtins import parameter, run_after # added only to make the linter happy -from reframe.utility import find_modules from hpctestlib.sciapps.gromacs.benchmarks import gromacs_check from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES from eessi.testsuite.eessi_mixin import EESSI_Mixin -from eessi.testsuite.utils import log +from eessi.testsuite.utils import find_modules, log class EESSI_GROMACS_base(gromacs_check): diff --git a/eessi/testsuite/tests/apps/lammps/lammps.py b/eessi/testsuite/tests/apps/lammps/lammps.py index ba6e3e32..4c13449e 100644 --- a/eessi/testsuite/tests/apps/lammps/lammps.py +++ b/eessi/testsuite/tests/apps/lammps/lammps.py @@ -53,7 +53,7 @@ class EESSI_LAMMPS_base(rfm.RunOnlyRegressionTest): device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) # Parameterize over all modules that start with LAMMPS - module_name = parameter(find_modules('LAMMPS')) + module_info = parameter(find_modules('LAMMPS')) all_readonly_files = True is_ci_test = True diff --git a/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py b/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py index 1125721b..5b1d0570 100644 --- a/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py +++ b/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py @@ -48,7 +48,7 @@ class EESSI_lbmpy_pssrt(rfm.RunOnlyRegressionTest, EESSI_Mixin): launcher = 'local' # no MPI module is loaded in this test - module_name = parameter(find_modules('lbmpy-pssrt')) + module_info = parameter(find_modules('lbmpy-pssrt')) readonly_files = ['mixing_layer_2D.py'] diff --git a/eessi/testsuite/tests/apps/lpc3d/lpc3d.py b/eessi/testsuite/tests/apps/lpc3d/lpc3d.py index ed972a41..b3250450 100644 --- a/eessi/testsuite/tests/apps/lpc3d/lpc3d.py +++ b/eessi/testsuite/tests/apps/lpc3d/lpc3d.py @@ -44,7 +44,7 @@ class EESSI_LPC3D(rfm.RunOnlyRegressionTest, EESSI_Mixin): launcher = 'local' # no MPI module is loaded in this test - module_name = parameter(find_modules('LPC3D')) + module_info = parameter(find_modules('LPC3D')) readonly_files = ['lattice_gas.inpt', 'pore_dens_freq_2neg.txt', 'psd.txt'] diff --git a/eessi/testsuite/tests/apps/numpy/numpy.py b/eessi/testsuite/tests/apps/numpy/numpy.py index 1c58cea4..8a288a99 100644 --- a/eessi/testsuite/tests/apps/numpy/numpy.py +++ b/eessi/testsuite/tests/apps/numpy/numpy.py @@ -12,11 +12,10 @@ import reframe as rfm import reframe.utility.sanity as sn from reframe.core.builtins import parameter, run_after, run_before, sanity_function, variable -from reframe.utility import find_modules from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES from eessi.testsuite.eessi_mixin import EESSI_Mixin - +from eessi.testsuite.utils import find_modules @rfm.simple_test class EESSI_NumPy(rfm.RunOnlyRegressionTest, EESSI_Mixin): diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index ad646893..ebf94790 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -87,7 +87,7 @@ class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_64M(rfm.RunOnlyRegressionTest, EESSI_Mixi time_limit = '120m' readonly_files = [''] device_type = parameter([DEVICE_TYPES.CPU]) - module_name = parameter(find_modules('OpenFOAM/v', name_only=False)) + module_info = parameter(find_modules('OpenFOAM/v', name_only=False)) valid_systems = ['*'] scale = parameter(filter_scales_64M()) diff --git a/eessi/testsuite/tests/apps/osu.py b/eessi/testsuite/tests/apps/osu.py index 9d7418f0..d6b44af3 100644 --- a/eessi/testsuite/tests/apps/osu.py +++ b/eessi/testsuite/tests/apps/osu.py @@ -57,7 +57,7 @@ def filter_scales_coll(): class EESSI_OSU_Base(osu_benchmark): """ base class for OSU tests """ time_limit = '30m' - module_name = parameter(find_modules('OSU-Micro-Benchmarks')) + module_info = parameter(find_modules('OSU-Micro-Benchmarks')) used_cpus_per_task = 1 # reset num_tasks_per_node from the hpctestlib: we handle it ourselves diff --git a/eessi/testsuite/tests/apps/tensorflow/tensorflow.py b/eessi/testsuite/tests/apps/tensorflow/tensorflow.py index a3b68bbe..09070f0d 100644 --- a/eessi/testsuite/tests/apps/tensorflow/tensorflow.py +++ b/eessi/testsuite/tests/apps/tensorflow/tensorflow.py @@ -19,7 +19,7 @@ class EESSI_TensorFlow(rfm.RunOnlyRegressionTest, EESSI_Mixin): # Parameterize over all modules that start with TensorFlow - module_name = parameter(find_modules('TensorFlow')) + module_info = parameter(find_modules('TensorFlow')) # Make CPU and GPU versions of this test device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) diff --git a/eessi/testsuite/tests/apps/walberla/walberla.py b/eessi/testsuite/tests/apps/walberla/walberla.py index 8112d8c2..9fbdcc85 100644 --- a/eessi/testsuite/tests/apps/walberla/walberla.py +++ b/eessi/testsuite/tests/apps/walberla/walberla.py @@ -44,7 +44,7 @@ class EESSI_WALBERLA_BACKWARD_FACING_STEP(rfm.RunOnlyRegressionTest, EESSI_Mixin time_limit = '30m' readonly_files = [''] device_type = parameter([DEVICE_TYPES.CPU]) - module_name = parameter(find_modules('waLBerla')) + module_info = parameter(find_modules('waLBerla')) valid_systems = ['*'] scale = parameter(filter_scales()) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index ef32b7c0..c0d10161 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -13,6 +13,7 @@ from reframe.core.logging import getlogger import reframe.core.runtime as rt from reframe.frontend.printer import PrettyPrinter +from reframe.utility import find_modules as rf_find_modules from eessi.testsuite.constants import DEVICE_TYPES @@ -145,67 +146,12 @@ def get_avail_modules() -> List[str]: return _available_modules -def find_modules(regex: str, name_only=True) -> Iterator[str]: +def find_modules(substr, environ_mapping=None) -> Iterator[tuple[str, str, str]]: """ - Return all modules matching the regular expression regex. Note that since we use re.search, - a module matches if the regex matches the module name at any place. I.e. the match does - not have to be at the start of the smodule name - - Arguments: - - regex: a regular expression - - name_only: regular expressions will only be matched on the module name, not the version (default: True). - - Note: the name_only feature assumes anything after the last forward '/' is the version, - and strips that before doing a match. - - Example - - Suppose we have the following modules on a system: - - gompic/2022a - gompi/2022a - CGAL/4.14.3-gompi-2022a - - The following calls would return the following respective modules - - find_modules('gompi') => [gompic/2022a, gompi/2022a] - find_modules('gompi$') => [gompi/2022a] - find_modules('gompi', name_only = False) => [gompic/2022a, gompi/2022a, CGAL/4.14.3-gompi-2022a] - find_modules('^gompi', name_only = False) => [gompic/2022a, gompi/2022a] - find_modules('^gompi/', name_only = False) => [gompi/2022a] - find_modules('-gompi-2022a', name_only = False) => [CGAL/4.14.3-gompi-2022a] - + Wraps reframe.utility.find_modules in order to provide caching, so that we don't have to do repeated + module avail calls. """ - - if not isinstance(regex, str): - raise TypeError("'substr' argument must be a string") - - seen = set() - dupes = [] - for mod in get_avail_modules(): - # The thing we yield should always be the original module name (orig_mod), including version - orig_mod = mod - if name_only: - # Remove trailing slashes from the regex (in case the callee forgot) - regex = regex.rstrip('/') - # Remove part after the last forward slash, as we assume this is the version - mod = re.sub('/[^/]*$', '', mod) - # Match the actual regular expression - log(f"Matching module {mod} with regex {regex}") - if re.search(regex, mod): - log("Match!") - if orig_mod in seen: - dupes.append(orig_mod) - else: - seen.add(orig_mod) - yield orig_mod - - if dupes: - err_msg = "EESSI test-suite cannot handle duplicate modules. " - err_msg += "Please make sure that only one is available on your system. " - err_msg += f"The following modules have a duplicate on your system: {dupes}" - raise ValueError(err_msg) - + return rf_find_modules(substr, environ_mapping) def get_tc_hierarchy(tcdict): """ From 8f7dad63324c2d27cd5dbac57192475da9940f08 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:38:39 +0100 Subject: [PATCH 03/12] Apply suggestion from @casparvl --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index c0d10161..b34463c6 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -6,7 +6,7 @@ import os import re import sys -from typing import Iterator, List +from typing import Iterator, List, Tuple import reframe as rfm from reframe.core.exceptions import ReframeFatalError From d6d4c5f57579214023e0aaed234c684a0bea1f1f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:38:54 +0100 Subject: [PATCH 04/12] Apply suggestion from @casparvl --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index b34463c6..1bf33b2b 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -146,7 +146,7 @@ def get_avail_modules() -> List[str]: return _available_modules -def find_modules(substr, environ_mapping=None) -> Iterator[tuple[str, str, str]]: +def find_modules(substr, environ_mapping=None) -> Iterator[Tuple[str, str, str]]: """ Wraps reframe.utility.find_modules in order to provide caching, so that we don't have to do repeated module avail calls. From 0e3a48a6ac418bec9419f5f280c2e0900d798cd5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 30 Mar 2026 16:30:35 +0200 Subject: [PATCH 05/12] Fix linter errors --- eessi/testsuite/eessi_mixin.py | 2 +- eessi/testsuite/tests/apps/numpy/numpy.py | 1 + eessi/testsuite/utils.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 6cc33fa6..138d74a7 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -184,7 +184,7 @@ def EESSI_mixin_run_after_init(self): # Filter by defice type. E.g. add features based on whether CUDA appears in the module name hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) - # Check if the partitions returned by find_modules satisfy the current features/extras specified in valid_systems + # Check if partitions returned by find_modules satisfy the current features/extras specified in valid_systems valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] if s in valid_partitions: self.valid_systems = [s] diff --git a/eessi/testsuite/tests/apps/numpy/numpy.py b/eessi/testsuite/tests/apps/numpy/numpy.py index 8a288a99..82b0229d 100644 --- a/eessi/testsuite/tests/apps/numpy/numpy.py +++ b/eessi/testsuite/tests/apps/numpy/numpy.py @@ -17,6 +17,7 @@ from eessi.testsuite.eessi_mixin import EESSI_Mixin from eessi.testsuite.utils import find_modules + @rfm.simple_test class EESSI_NumPy(rfm.RunOnlyRegressionTest, EESSI_Mixin): descr = 'Test matrix operations with NumPy' diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index c0d10161..558abb51 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -151,8 +151,10 @@ def find_modules(substr, environ_mapping=None) -> Iterator[tuple[str, str, str]] Wraps reframe.utility.find_modules in order to provide caching, so that we don't have to do repeated module avail calls. """ + # TODO: implement caching to make this function more efficient return rf_find_modules(substr, environ_mapping) + def get_tc_hierarchy(tcdict): """ Set up EasyBuild configuration and get toolchain hierarchy from a toolchain dict From faf21ae75adcddbd7689f3e31ce4f26ab0cafba9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 2 Apr 2026 16:57:30 +0200 Subject: [PATCH 06/12] Make sure that if reframe is new enough, we simply use the combination of sys:part +feat notation directly --- eessi/testsuite/eessi_mixin.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 138d74a7..0c3be8bd 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -10,6 +10,7 @@ from reframe.utility.sanity import make_performance_function import reframe.utility.sanity as sn from reframe.core.runtime import valid_sysenv_comb +from reframe import VERSION as reframe_version from eessi.testsuite import check_process_binding, hooks from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES, TAGS @@ -181,15 +182,33 @@ def EESSI_mixin_run_after_init(self): # Set modules hooks.set_modules(self) - # Filter by defice type. E.g. add features based on whether CUDA appears in the module name - hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) - - # Check if partitions returned by find_modules satisfy the current features/extras specified in valid_systems - valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] - if s in valid_partitions: + # Checks reframe version, and if newer or equal to 4.10, use the new functionality + # that allows combining sys:part notation with +feat. + syspart_feat_supported = False + try: + import semver + semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0") + syspart_feat_supported = True + except: + pass + + # If we use reframe 4.10.0 or later, we can just set the valid system and the hook will + # append any relevant features. Otherwise, as a fallback, we set the features first + # (by calling the hook), then check if the sys:part combination from the find_modules triplet + # is in the list of valid combinations for the given features + if syspart_feat_supported: self.valid_systems = [s] + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) else: - self.valid_systems = [] + # Filter by defice type. E.g. add features based on whether CUDA appears in the module name + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + + # Check if partitions returned by find_modules satisfy the current features/extras specified in valid_systems + valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] + if s in valid_partitions: + self.valid_systems = [s] + else: + self.valid_systems = [] # Set scales as tags hooks.set_tag_scale(self) From 8303df6ec871a4ff2263278f46737d83479a1dd5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 2 Apr 2026 17:00:44 +0200 Subject: [PATCH 07/12] Fix linting issues --- eessi/testsuite/eessi_mixin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 0c3be8bd..5aaad08a 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -189,7 +189,7 @@ def EESSI_mixin_run_after_init(self): import semver semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0") syspart_feat_supported = True - except: + except ImportError: pass # If we use reframe 4.10.0 or later, we can just set the valid system and the hook will @@ -203,7 +203,7 @@ def EESSI_mixin_run_after_init(self): # Filter by defice type. E.g. add features based on whether CUDA appears in the module name hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) - # Check if partitions returned by find_modules satisfy the current features/extras specified in valid_systems + # Check if partitions returned by find_modules satisfy the current features/extras in valid_systems valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] if s in valid_partitions: self.valid_systems = [s] From 5d588334bf384eff558303784edd3e840a3e42d0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:08:27 +0200 Subject: [PATCH 08/12] add missing 'if' Co-authored-by: Sam Moors --- eessi/testsuite/eessi_mixin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 5aaad08a..a37b6259 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -187,8 +187,8 @@ def EESSI_mixin_run_after_init(self): syspart_feat_supported = False try: import semver - semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0") - syspart_feat_supported = True + if semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0"): + syspart_feat_supported = True except ImportError: pass From 63c0eabfa16b44c96d7a93c18bce7d7f127ea6e2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 Apr 2026 17:11:01 +0200 Subject: [PATCH 09/12] Use at least three letters for variable assignments --- eessi/testsuite/eessi_mixin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index a37b6259..6436dc9d 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -175,9 +175,9 @@ def EESSI_mixin_run_after_init(self): raise EESSIError(err_msg) # Unpack module_info - s, e, m = self.module_info - self.valid_prog_environs = [e] - self.module_name = [m] + sys, env, mod = self.module_info + self.valid_prog_environs = [env] + self.module_name = [mod] # Set modules hooks.set_modules(self) @@ -197,16 +197,16 @@ def EESSI_mixin_run_after_init(self): # (by calling the hook), then check if the sys:part combination from the find_modules triplet # is in the list of valid combinations for the given features if syspart_feat_supported: - self.valid_systems = [s] + self.valid_systems = [sys] hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) else: # Filter by defice type. E.g. add features based on whether CUDA appears in the module name hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) # Check if partitions returned by find_modules satisfy the current features/extras in valid_systems - valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, e)] - if s in valid_partitions: - self.valid_systems = [s] + valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, env)] + if sys in valid_partitions: + self.valid_systems = [sys] else: self.valid_systems = [] From 0112ebeee497817695b206867659742891c7eecf Mon Sep 17 00:00:00 2001 From: casparvl Date: Wed, 29 Apr 2026 13:51:44 +0000 Subject: [PATCH 10/12] Make appending to the existing environment the default, so that one can still load environments for testing local modules --- eessi/testsuite/common_config.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py index 71022668..45dd69e2 100644 --- a/eessi/testsuite/common_config.py +++ b/eessi/testsuite/common_config.py @@ -61,16 +61,23 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True }] if 'environments' in site_configuration and site_configuration['environments'] != environments: - getlogger().info(f"Changing environments in site config to {environments}") - site_configuration['environments'] = environments + msg = f"Appending environments {environments} to the environments already present in the site_configuration" + msg += f" ({site_configuration['environments']})" + getlogger().info(msg) + site_configuration['environments'].extend(environments) + else: + site_configuration['environments'] = environments for system in site_configuration.get('systems', []): for partition in system.get('partitions', []): # Set or overwrite the partition environment if 'environs' in partition and partition['environs'] != environs: - getlogger().info( - f"Changing environs in site config to {environs} for {system['name']}:{partition['name']}") - partition['environs'] = environs + msg = f"Appending environs {environs} to the existing environs ({partition['environs']})" + msg += f" for {system['name']}:{partition['name']}" + getlogger().info(msg) + partition['environs'].extend(environs) + else: + partition['environs'] = environs # Set or overwrite the 'use_nodes_option' scheduler option, if this is a SLURM-like scheduler if partition['scheduler'] in ['slurm', 'squeue']: From 3f4edfac5a16f552c528858758f6be696804f842 Mon Sep 17 00:00:00 2001 From: casparvl Date: Wed, 29 Apr 2026 15:01:53 +0000 Subject: [PATCH 11/12] No longer require that the reframe environment is always called 'default', as we now use ReFrame environments to test over multiple versions of EESSI --- eessi/testsuite/eessi_mixin.py | 3 +-- eessi/testsuite/tests/apps/MetalWalls.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 6436dc9d..5ae018bc 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -88,7 +88,7 @@ class EESSI_Mixin(RegressionTestPlugin): def __init_subclass__(cls, **kwargs): " set default values for built-in ReFrame attributes " super().__init_subclass__(**kwargs) - cls.valid_prog_environs = ['default'] + cls.valid_prog_environs = ['*'] cls.valid_systems = ['*'] if not cls.time_limit: cls.time_limit = '1h' @@ -151,7 +151,6 @@ def EESSI_mixin_validate_init(self): self.EESSI_mixin_validate_item_in_list('device_type', DEVICE_TYPES[:]) self.EESSI_mixin_validate_item_in_list('scale', SCALES.keys()) self.EESSI_mixin_validate_item_in_list('valid_systems', [['*']]) - self.EESSI_mixin_validate_item_in_list('valid_prog_environs', [['default']]) @run_after('init') def EESSI_mixin_run_after_init(self): diff --git a/eessi/testsuite/tests/apps/MetalWalls.py b/eessi/testsuite/tests/apps/MetalWalls.py index fc4c8cb5..7e31a070 100644 --- a/eessi/testsuite/tests/apps/MetalWalls.py +++ b/eessi/testsuite/tests/apps/MetalWalls.py @@ -46,7 +46,7 @@ class EESSI_MetalWalls_MW(MetalWallsCheck): scale = parameter(SCALES.keys()) valid_systems = ['*'] - valid_prog_environs = ['default'] + valid_prog_environs = ['*'] time_limit = '60m' # input files are downloaded readonly_files = [''] From 2a6ddbb4fe187bef7e71d96abcfd2bb7bbb8f70f Mon Sep 17 00:00:00 2001 From: casparvl Date: Wed, 29 Apr 2026 15:23:18 +0000 Subject: [PATCH 12/12] Rely on the common config to set the environs --- config/github_actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/config/github_actions.py b/config/github_actions.py index adb92e3e..3bf0c878 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -16,7 +16,6 @@ 'name': 'default', 'scheduler': 'local', 'launcher': 'local', - 'environs': ['default'], 'features': [FEATURES.CPU] + list(SCALES.keys()), 'processor': { 'num_cpus': 2,