diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f195df0a..2765d27f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - name: Run test suite run: | - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/lmod/bash module load ReFrame reframe --version diff --git a/config/settings_example.py b/config/settings_example.py index 7c4ea87d..a34ad27d 100644 --- a/config/settings_example.py +++ b/config/settings_example.py @@ -116,9 +116,11 @@ ], } -# Set prepare commands for making EESSI modules available +# Set prepare commands for making EESSI modules available in the test jobs. +# Note: make sure that the EESSI modules are also available (in $MODULEPATH) before starting ReFrame: +# the prepare commands only run in the test jobs, while EESSI versions and software are detected at startup. eessi_prepare_cmds = [ - 'module unuse MODULEPATH', + 'module unuse $MODULEPATH', 'module use /cvmfs/software.eessi.io/init/modules', # If the system doesn’t have an Lmod installation by default, the following # commands will use the one from EESSI. Note that it is up to you to pick a @@ -129,4 +131,11 @@ ] # Set common Slurm config options +# By default, all EESSI versions found by Lmod are added as environments to every partition. +# To change this for all partitions, use the add_eessi_versions argument: +# - add_eessi_versions=['2025.06']: only add these versions +# - add_eessi_versions=False: add no versions, define the EESSI environments and partition environs yourself +# To change this for a single partition, set its environs: +# - 'environs': ['EESSI-2025.06']: only use these EESSI versions +# - 'environs': ['my_env', NO_EESSI]: don't add EESSI environments (NO_EESSI is in eessi.testsuite.constants) set_common_required_config(site_configuration, eessi_prepare_cmds=eessi_prepare_cmds) diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py index c9011713..81481bfd 100644 --- a/eessi/testsuite/common_config.py +++ b/eessi/testsuite/common_config.py @@ -2,10 +2,11 @@ import os from reframe.core.config import load_config -from reframe.core.exceptions import ConfigError +from reframe.core.exceptions import ConfigError, EnvironError from reframe.core.logging import getlogger +from reframe.core.modules import ModulesSystem -from eessi.testsuite.constants import FEATURES +from eessi.testsuite.constants import FEATURES, NO_EESSI from eessi.testsuite.utils import EESSIError @@ -77,21 +78,72 @@ def reframe_version_is_supported(): ]) -def set_common_required_config(site_configuration: dict, set_memory: bool = True, eessi_prepare_cmds=None): +def get_available_eessi_versions(): + """ + Query Lmod for the available EESSI versions. + The EESSI modules must already be available (in $MODULEPATH) before running ReFrame. + :return: sorted list of available EESSI versions, e.g. ['2023.06', '2025.06'] + """ + # EESSI modules are Lua modulefiles, so they can only be found and used with Lmod + try: + lmod = ModulesSystem.create('lmod') + # EESSI modules may be hidden by the site, so include hidden modules + output = lmod.execute('--show-hidden', '--terse', 'avail', 'EESSI') + except (ConfigError, EnvironError) as err: + getlogger().warning(f'Not adding EESSI environments: Lmod is required to use EESSI, but failed with: {err}') + return [] + + # lines look like 'EESSI/2025.06' or 'EESSI/2025.06 ' + versions = { + line.split()[0].split('/', 1)[1] + for line in output.splitlines() + if line.startswith('EESSI/') + } + if not versions: + getlogger().warning( + 'No EESSI versions found. If you want to test the EESSI software stack,' + ' make sure the EESSI modules are available before running ReFrame.' + ) + return sorted(versions) + + +def set_common_required_config(site_configuration: dict, set_memory: bool = True, eessi_prepare_cmds=None, + add_eessi_versions=True): """ Update ReFrame configuration file: set common required config options. Must be called at the end of the site configuration file (below the `site_configuration` dict). :param site_configuration: site configuration dictionary :param set_memory: whether to set memory resources + :param eessi_prepare_cmds: list of prepare commands for the environments created for `add_eessi_versions` + :param add_eessi_versions: EESSI versions to add to the site configuration, each as a ReFrame environment + that is also added to the partitions. Environments already defined by the site are kept as they are. + A partition that lists no EESSI environment gets all added versions; a partition that lists one or more + (name starting with 'EESSI-') is left alone, so it runs exactly those. To add no EESSI environments to a + partition, add the marker `NO_EESSI` (from eessi.testsuite.constants) to its 'environs'. + True (default): all versions found by Lmod + list of versions: only those versions, e.g. ['2025.06'] + False: add no versions, the site defines its own EESSI environments and partition environs """ if not eessi_prepare_cmds: eessi_prepare_cmds = [] + if add_eessi_versions is True: + eessi_versions = get_available_eessi_versions() + elif add_eessi_versions is False: + eessi_versions = [] + elif isinstance(add_eessi_versions, (list, tuple)): + eessi_versions = add_eessi_versions + else: + raise EESSIError(f'add_eessi_versions must be True, False or a list of versions, got {add_eessi_versions!r}') + + if eessi_prepare_cmds and not add_eessi_versions: + getlogger().warning('Ignoring eessi_prepare_cmds: add_eessi_versions is set to add no EESSI versions.') + environments = [ - {'name': 'EESSI-2023.06', 'modules': ['EESSI/2023.06'], 'prepare_cmds': eessi_prepare_cmds}, - {'name': 'EESSI-2025.06', 'modules': ['EESSI/2025.06'], 'prepare_cmds': eessi_prepare_cmds}, + {'name': f'EESSI-{version}', 'modules': [f'EESSI/{version}'], 'prepare_cmds': eessi_prepare_cmds} + for version in eessi_versions ] - environs = ['EESSI-2023.06', 'EESSI-2025.06'] + environs = [env['name'] for env in environments] use_nodes_option = True if set_memory: resources_memory = [{ @@ -105,24 +157,55 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True 'options': ['--gpus-per-node={num_gpus_per_node}'], }] - if 'environments' in site_configuration and site_configuration['environments'] != environments: - msg = f"Appending environments {environments} to the environments already present in the site_configuration" - msg += f" ({site_configuration['environments']})" - getlogger().info(msg) - site_configuration['environments'].extend(environments) - else: + if 'environments' not in site_configuration: site_configuration['environments'] = environments + else: + # Don't redefine environments that are already defined by the site + defined_environments = {env['name'] for env in site_configuration['environments']} + new_environments = [env for env in environments if env['name'] not in defined_environments] + kept_environments = [env['name'] for env in environments if env['name'] in defined_environments] + if kept_environments: + getlogger().info(f"Using site-defined environments {kept_environments}") + if new_environments: + msg = f"Appending environments {[env['name'] for env in new_environments]} to the environments" + msg += f" already present in the site_configuration ({sorted(defined_environments)})" + getlogger().info(msg) + site_configuration['environments'].extend(new_environments) for system in site_configuration.get('systems', []): for partition in system.get('partitions', []): - # Set or overwrite the partition environment - if 'environs' in partition and partition['environs'] != environs: - msg = f"Appending environs {environs} to the existing environs ({partition['environs']})" - msg += f" for {system['name']}:{partition['name']}" + # Add the EESSI environments, unless the site already selected specific EESSI environments for the partition + # or opted out with the NO_EESSI marker. The marker is not a real environment, so it must always be removed. + if NO_EESSI in partition.get('environs', []): + partition['environs'] = [env for env in partition['environs'] if env != NO_EESSI] + if any(env.startswith('EESSI-') for env in partition['environs']): + msg = f"{system['name']}:{partition['name']} lists both {NO_EESSI} and EESSI environments:" + msg += f" {partition['environs']}" + raise EESSIError(msg) + msg = f"Not adding EESSI environs to {system['name']}:{partition['name']}:" + msg += f" it has the {NO_EESSI} marker in its environs" getlogger().info(msg) - partition['environs'].extend(environs) - else: - partition['environs'] = environs + if not partition['environs']: + msg = f"No environs set for {system['name']}:{partition['name']}:" + msg += " no tests will run on this partition." + getlogger().warning(msg) + elif 'environs' not in partition: + partition['environs'] = list(environs) + if not add_eessi_versions: + msg = f"No environs set for {system['name']}:{partition['name']}:" + msg += " no tests will run on this partition." + getlogger().warning(msg) + elif environs: + existing_eessi_environs = [env for env in partition['environs'] if env.startswith('EESSI-')] + if existing_eessi_environs: + msg = f"Not adding more EESSI environs to {system['name']}:{partition['name']}:" + msg += f" it already lists {existing_eessi_environs}" + getlogger().info(msg) + else: + msg = f"Appending environs {environs} to the existing environs ({partition['environs']})" + msg += f" for {system['name']}:{partition['name']}" + getlogger().info(msg) + partition['environs'] = partition['environs'] + environs # Set or overwrite the 'use_nodes_option' scheduler option, if this is a SLURM-like scheduler if partition['scheduler'] in ['slurm', 'squeue']: diff --git a/eessi/testsuite/constants.py b/eessi/testsuite/constants.py index c198e493..625938af 100644 --- a/eessi/testsuite/constants.py +++ b/eessi/testsuite/constants.py @@ -53,6 +53,10 @@ class _GPUVendors(NamedTuple): FEATURES = _Features() GPU_VENDORS = _GPUVendors() +# Marker for the 'environs' of a partition in the site configuration: don't add EESSI environments to this partition. +# It is removed from 'environs' by set_common_required_config, so it is not a real ReFrame environment. +NO_EESSI = 'NO_EESSI' + SCALES = { # required keys: # - num_nodes