Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:

- name: Run test suite
run: |
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/lmod/bash

module load ReFrame
reframe --version
Expand Down
13 changes: 11 additions & 2 deletions config/settings_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,11 @@
],
}

# Set prepare commands for making EESSI modules available
# Set prepare commands for making EESSI modules available in the test jobs.
# Note: make sure that the EESSI modules are also available (in $MODULEPATH) before starting ReFrame:
# the prepare commands only run in the test jobs, while EESSI versions and software are detected at startup.
eessi_prepare_cmds = [
'module unuse MODULEPATH',
'module unuse $MODULEPATH',
'module use /cvmfs/software.eessi.io/init/modules',
# If the system doesn’t have an Lmod installation by default, the following
# commands will use the one from EESSI. Note that it is up to you to pick a
Expand All @@ -129,4 +131,11 @@
]

# Set common Slurm config options
# By default, all EESSI versions found by Lmod are added as environments to every partition.
# To change this for all partitions, use the add_eessi_versions argument:
# - add_eessi_versions=['2025.06']: only add these versions
# - add_eessi_versions=False: add no versions, define the EESSI environments and partition environs yourself
# To change this for a single partition, set its environs:
# - 'environs': ['EESSI-2025.06']: only use these EESSI versions
# - 'environs': ['my_env', NO_EESSI]: don't add EESSI environments (NO_EESSI is in eessi.testsuite.constants)
set_common_required_config(site_configuration, eessi_prepare_cmds=eessi_prepare_cmds)
121 changes: 102 additions & 19 deletions eessi/testsuite/common_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@
import os

from reframe.core.config import load_config
from reframe.core.exceptions import ConfigError
from reframe.core.exceptions import ConfigError, EnvironError
from reframe.core.logging import getlogger
from reframe.core.modules import ModulesSystem

from eessi.testsuite.constants import FEATURES
from eessi.testsuite.constants import FEATURES, NO_EESSI
from eessi.testsuite.utils import EESSIError


Expand Down Expand Up @@ -77,21 +78,72 @@ def reframe_version_is_supported():
])


def set_common_required_config(site_configuration: dict, set_memory: bool = True, eessi_prepare_cmds=None):
def get_available_eessi_versions():
"""
Query Lmod for the available EESSI versions.
The EESSI modules must already be available (in $MODULEPATH) before running ReFrame.
:return: sorted list of available EESSI versions, e.g. ['2023.06', '2025.06']
"""
# EESSI modules are Lua modulefiles, so they can only be found and used with Lmod
try:
lmod = ModulesSystem.create('lmod')
# EESSI modules may be hidden by the site, so include hidden modules
output = lmod.execute('--show-hidden', '--terse', 'avail', 'EESSI')
except (ConfigError, EnvironError) as err:
getlogger().warning(f'Not adding EESSI environments: Lmod is required to use EESSI, but failed with: {err}')
return []

# lines look like 'EESSI/2025.06' or 'EESSI/2025.06 <H>'
versions = {
line.split()[0].split('/', 1)[1]
for line in output.splitlines()
if line.startswith('EESSI/')
}
if not versions:
getlogger().warning(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a bit inconsistent: if we can't detect EESSI versions because there is no LMOD_CMD, it's a hard error. If we can't detect EESSI versions because there is a module command, but your cmd has simply not returned any EESS modules (e.g. because the MODULEPATH wasn't set correctly) it's a warning. We should make this consistent, but... with what?

You could say: it's an error, because the user specified add_eessi_versions=True in their config (well, implicitely, since it is the default). If you can't detect EESSI environments, we cannot add them, and since that was requested, it is an error.

One could also argue that this is a breaking change, and that raising an error would be 'too breaking'. I.e. without changing any of the defaults, if my MODULEPATH doesn't contain EESSI (or if I don't have a modulecommand), I now suddenly get a hard error.

I'm not sure which way to go...

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there's probably no clear answer, but the way i see it:

  • if there is not module system available (assuming we'll generalize LMOD_CMD to other module systems), then we can't test any modules at all, so should clearly error.
  • if there are no EESSI versions detected, we can still test local modules if there is a local environ defined, so it's not as bad.

a solution could be to add a ReFram variable, say no_eessi_modules_found_policy, similar to required_mem_per_node_undefined_policy and readonly_files_undefined_policy. then each user can decide if they want an error or a warning.

what do you think?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm, maybe I'm just bikeshedding. I was really thinking about it from a single function perspective: if you ahve a function that is supposed to 'get' the EESSI versions, and it didn't find any, I thought that was a (hard) failure as well. I.e. function didn't return what it was supposed to return. But I guess that's not the case: it did return what it was supposed to return, namely 'there are no EESSI modules on this system' (but we can still test the local ones).

Fine, let's just keep it like this - maybe generalize the LMOD_CMD to other module systems. I do still think that a hard error on a system that has Environment Modules instead of Lmod is... too harsh.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anyway, this conversation is resolve,d we'll pick up the LMOD_CMD thing at that line :)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changed it to use warning for the missing Lmod, since other module systems may stil be available and so local modules can still be tested. so both are now a warning. also using the ReFrame machinery for the modules sytem now.

7bd05a3

'No EESSI versions found. If you want to test the EESSI software stack,'
' make sure the EESSI modules are available before running ReFrame.'
)
return sorted(versions)


def set_common_required_config(site_configuration: dict, set_memory: bool = True, eessi_prepare_cmds=None,
add_eessi_versions=True):
"""
Update ReFrame configuration file: set common required config options.
Must be called at the end of the site configuration file (below the `site_configuration` dict).
:param site_configuration: site configuration dictionary
:param set_memory: whether to set memory resources
:param eessi_prepare_cmds: list of prepare commands for the environments created for `add_eessi_versions`
:param add_eessi_versions: EESSI versions to add to the site configuration, each as a ReFrame environment
that is also added to the partitions. Environments already defined by the site are kept as they are.
A partition that lists no EESSI environment gets all added versions; a partition that lists one or more
(name starting with 'EESSI-') is left alone, so it runs exactly those. To add no EESSI environments to a
partition, add the marker `NO_EESSI` (from eessi.testsuite.constants) to its 'environs'.
True (default): all versions found by Lmod
list of versions: only those versions, e.g. ['2025.06']
False: add no versions, the site defines its own EESSI environments and partition environs
"""
if not eessi_prepare_cmds:
eessi_prepare_cmds = []

if add_eessi_versions is True:
eessi_versions = get_available_eessi_versions()
elif add_eessi_versions is False:
eessi_versions = []
elif isinstance(add_eessi_versions, (list, tuple)):
eessi_versions = add_eessi_versions
else:
raise EESSIError(f'add_eessi_versions must be True, False or a list of versions, got {add_eessi_versions!r}')

if eessi_prepare_cmds and not add_eessi_versions:
getlogger().warning('Ignoring eessi_prepare_cmds: add_eessi_versions is set to add no EESSI versions.')

environments = [
{'name': 'EESSI-2023.06', 'modules': ['EESSI/2023.06'], 'prepare_cmds': eessi_prepare_cmds},
{'name': 'EESSI-2025.06', 'modules': ['EESSI/2025.06'], 'prepare_cmds': eessi_prepare_cmds},
{'name': f'EESSI-{version}', 'modules': [f'EESSI/{version}'], 'prepare_cmds': eessi_prepare_cmds}
for version in eessi_versions
]
environs = ['EESSI-2023.06', 'EESSI-2025.06']
environs = [env['name'] for env in environments]
use_nodes_option = True
if set_memory:
resources_memory = [{
Expand All @@ -105,24 +157,55 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True
'options': ['--gpus-per-node={num_gpus_per_node}'],
}]

if 'environments' in site_configuration and site_configuration['environments'] != environments:
msg = f"Appending environments {environments} to the environments already present in the site_configuration"
msg += f" ({site_configuration['environments']})"
getlogger().info(msg)
site_configuration['environments'].extend(environments)
else:
if 'environments' not in site_configuration:
site_configuration['environments'] = environments
else:
# Don't redefine environments that are already defined by the site
defined_environments = {env['name'] for env in site_configuration['environments']}
new_environments = [env for env in environments if env['name'] not in defined_environments]
kept_environments = [env['name'] for env in environments if env['name'] in defined_environments]
if kept_environments:
getlogger().info(f"Using site-defined environments {kept_environments}")
if new_environments:
msg = f"Appending environments {[env['name'] for env in new_environments]} to the environments"
msg += f" already present in the site_configuration ({sorted(defined_environments)})"
getlogger().info(msg)
site_configuration['environments'].extend(new_environments)

for system in site_configuration.get('systems', []):
for partition in system.get('partitions', []):
# Set or overwrite the partition environment
if 'environs' in partition and partition['environs'] != environs:
msg = f"Appending environs {environs} to the existing environs ({partition['environs']})"
msg += f" for {system['name']}:{partition['name']}"
# Add the EESSI environments, unless the site already selected specific EESSI environments for the partition
# or opted out with the NO_EESSI marker. The marker is not a real environment, so it must always be removed.
if NO_EESSI in partition.get('environs', []):
partition['environs'] = [env for env in partition['environs'] if env != NO_EESSI]
if any(env.startswith('EESSI-') for env in partition['environs']):
msg = f"{system['name']}:{partition['name']} lists both {NO_EESSI} and EESSI environments:"
msg += f" {partition['environs']}"
raise EESSIError(msg)
msg = f"Not adding EESSI environs to {system['name']}:{partition['name']}:"
msg += f" it has the {NO_EESSI} marker in its environs"
getlogger().info(msg)
partition['environs'].extend(environs)
else:
partition['environs'] = environs
if not partition['environs']:
msg = f"No environs set for {system['name']}:{partition['name']}:"
msg += " no tests will run on this partition."
getlogger().warning(msg)
elif 'environs' not in partition:
partition['environs'] = list(environs)
if not add_eessi_versions:
msg = f"No environs set for {system['name']}:{partition['name']}:"
msg += " no tests will run on this partition."
getlogger().warning(msg)
elif environs:
existing_eessi_environs = [env for env in partition['environs'] if env.startswith('EESSI-')]
if existing_eessi_environs:
msg = f"Not adding more EESSI environs to {system['name']}:{partition['name']}:"
msg += f" it already lists {existing_eessi_environs}"
getlogger().info(msg)
else:
msg = f"Appending environs {environs} to the existing environs ({partition['environs']})"
msg += f" for {system['name']}:{partition['name']}"
getlogger().info(msg)
partition['environs'] = partition['environs'] + environs

# Set or overwrite the 'use_nodes_option' scheduler option, if this is a SLURM-like scheduler
if partition['scheduler'] in ['slurm', 'squeue']:
Expand Down
4 changes: 4 additions & 0 deletions eessi/testsuite/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ class _GPUVendors(NamedTuple):
FEATURES = _Features()
GPU_VENDORS = _GPUVendors()

# Marker for the 'environs' of a partition in the site configuration: don't add EESSI environments to this partition.
# It is removed from 'environs' by set_common_required_config, so it is not a real ReFrame environment.
NO_EESSI = 'NO_EESSI'

SCALES = {
# required keys:
# - num_nodes
Expand Down
Loading