Skip to content

Commit f58ed13

Browse files
bedrogebedroge
committed
Merge branch 'main' of github.com:EESSI/software-layer-scripts into zstd_tarballs
2 parents fc5d5ad + 6303643 commit f58ed13

File tree

13 files changed

+459
-167
lines changed

13 files changed

+459
-167
lines changed

‎.github/workflows/scripts/test_init_scripts.sh‎

Lines changed: 59 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,59 +2,81 @@
22
export LMOD_PAGER=cat
33

44
if [ -z ${EESSI_VERSION} ] || [ ! -d /cvmfs/software.eessi.io/versions/${EESSI_VERSION} ]; then
5-
echo "\$EESSI_VERSION has to be set to a valid EESSI version."
6-
exit 1
5+
echo "\$EESSI_VERSION has to be set to a valid EESSI version."
6+
exit 1
77
fi
88

99
if [ -z ${EXPECTED_EASYBUILD_VERSION} ]; then
10-
echo "\$EXPECTED_EASYBUILD_VERSION has to be set to an EasyBuild version that is expected to be available in EESSI version ${EESSI_VERSION}."
11-
exit 1
10+
echo "\$EXPECTED_EASYBUILD_VERSION has to be set to an EasyBuild version that is expected to be available in EESSI version ${EESSI_VERSION}."
11+
exit 1
1212
fi
1313

1414
# initialize assert framework
1515
if [ ! -d assert.sh ]; then
16-
echo "assert.sh not cloned."
17-
echo ""
18-
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
19-
exit 1
16+
echo "assert.sh not cloned."
17+
echo ""
18+
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
19+
exit 1
2020
fi
2121
. assert.sh/assert.sh
2222

23-
TEST_SHELLS=("bash" "zsh" "fish" "ksh")
23+
TEST_SHELLS=("bash" "zsh" "fish" "ksh" "csh")
2424
SHELLS=$@
2525

2626
for shell in ${SHELLS[@]}; do
27-
echo = | awk 'NF += (OFS = $_) + 100'
28-
echo RUNNING TESTS FOR SHELL: $shell
29-
echo = | awk 'NF += (OFS = $_) + 100'
27+
echo = | awk 'NF += (OFS = $_) + 100'
28+
echo RUNNING TESTS FOR SHELL: $shell
29+
echo = | awk 'NF += (OFS = $_) + 100'
3030
if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
31-
### EXCEPTION FOR CSH ###
32-
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
31+
### EXCEPTION FOR CSH ###
32+
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
3333
else
34-
# TEST 1: Source Script and check Module Output
35-
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "Module for EESSI/$EESSI_VERSION loaded successfully"
36-
# TEST 2: Check if module overviews first section is the loaded EESSI module
37-
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
38-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
39-
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
40-
# TEST 3: Check if module overviews second section is the EESSI init module
41-
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
42-
# Test 4: Load EasyBuild module and check version
43-
# eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
44-
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
45-
assert "$command" "$EXPECTED_EASYBUILD_VERSION"
46-
# Test 5: Load EasyBuild module and check path
47-
EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
48-
# escape the dots in ${EASYBUILD_VERSION}
49-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
50-
echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
51-
assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
52-
53-
#End Test Suite
54-
assert_end "source_eessi_$shell"
55-
fi
56-
done
34+
# TEST 1: Source Script and check Module Output
35+
expected="Module for EESSI/$EESSI_VERSION loaded successfully"
36+
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "${expected}"
37+
38+
# TEST 2: Check if module overviews first section is the loaded EESSI module
39+
if [ "$shell" = "csh" ]; then
40+
# module is defined as alias, but aliases are only retained in interactive
41+
# shells we work around this by creating a .cshrc file (which sources the
42+
# init script), and then simply run the remaining commands
43+
echo "source init/lmod/$shell" > ~/.cshrc
44+
MODULE_SECTIONS=($($shell -c "module ov" 2>&1 | grep -e '---'))
45+
else
46+
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
47+
fi
48+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
49+
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
50+
51+
# TEST 3: Check if module overviews second section is the EESSI init module
52+
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
5753

54+
# TEST 4: Load EasyBuild module and check version
55+
# eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
56+
if [ "$shell" = "csh" ]; then
57+
echo "source init/lmod/$shell" > ~/.cshrc
58+
command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | cut -d \" \" -f4"
59+
else
60+
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
61+
fi
62+
assert "$command" "$EXPECTED_EASYBUILD_VERSION"
63+
64+
# TEST 5: Load EasyBuild module and check path
65+
if [ "$shell" = "csh" ]; then
66+
echo "source init/lmod/$shell" > ~/.cshrc
67+
EASYBUILD_PATH=$($shell -c "module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
68+
else
69+
EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
70+
fi
71+
# escape the dots in ${EASYBUILD_VERSION}
72+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
73+
echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
74+
assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
75+
76+
# End Test Suite
77+
assert_end "source_eessi_$shell"
78+
fi
79+
done
5880

5981
# RESET PAGER
6082
export LMOD_PAGER=

‎.github/workflows/tests_init_module.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ jobs:
2727
- name: Check out software-layer repository
2828
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
2929

30+
- name: Avoid processing man-db trigger
31+
run: |
32+
# avoid processing trigger for man-db (seems to slow down installation)
33+
echo "set man-db/auto-update false" | sudo debconf-communicate
34+
sudo dpkg-reconfigure man-db
35+
3036
- name: Mount EESSI CernVM-FS pilot repository
3137
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
3238
with:
@@ -46,7 +52,7 @@ jobs:
4652
- name: Install missing shells
4753
run: |
4854
sudo apt update
49-
sudo apt install zsh ksh fish
55+
sudo apt install zsh ksh fish tcsh
5056
echo "# INIT ZSH" > ~/.zshrc
5157
5258
- name: Run tests for available shells

‎EESSI-extend-easybuild.eb‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,16 @@ description = """
4545
toolchain = SYSTEM
4646

4747
# All the dependencies we filter in EESSI
48-
local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,ParMETIS,util-linux,XZ,zlib"
48+
local_deps_to_filter = "binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,makeinfo,ncurses,ParMETIS,util-linux,XZ,zlib"
49+
50+
# filter extra dependencies based on CPU family
4951
local_arch_specific_deps_to_filter = {'aarch64': ',Yasm', 'riscv64': ',Yasm', 'x86_64': ''}
5052
local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH]
5153

54+
# only filter Autotools & co for EESSI 2023.06
55+
if version == '2023.06':
56+
local_deps_to_filter += ",Autoconf,Automake,Autotools,libtool,M4"
57+
5258
# Set the universal EasyBuild variables
5359
modextravars = {
5460
'EASYBUILD_FILTER_DEPS': local_deps_to_filter,
@@ -170,7 +176,8 @@ end
170176
setenv ("EASYBUILD_SYSROOT", sysroot)
171177
setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild"))
172178
setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath)
173-
setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py"))
179+
eessi_init_prefix = os.getenv("EESSI_INIT_PREFIX") or pathJoin(os.getenv("EESSI_PREFIX"), "init")
180+
setenv ("EASYBUILD_HOOKS", pathJoin(eessi_init_prefix, "easybuild", "eb_hooks.py"))
174181
175182
-- Make sure to use the general umask that allows a global read
176183
setenv ("EASYBUILD_UMASK", "022")
@@ -216,7 +223,7 @@ eessi_version = os.getenv("EESSI_VERSION") or "2023.06"
216223
-- This avoids issues where EESSI-extend is first loaded with EB => 5.1 (which set these vars)
217224
-- but then EB is swapped for a version < 5.1 and then EESSI-extend is unloaded (which would not unset
218225
-- these vars if we did it conditional on the EB version)
219-
if convertToCanonical(easybuild_version) >= convertToCanonical("5.1") or mode() == "unload" then
226+
if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuild_version) >= convertToCanonical("5.1") then
220227
setenv ("EASYBUILD_STRICT_RPATH_SANITY_CHECK", "1")
221228
setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
222229
setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")
@@ -228,6 +235,13 @@ if convertToCanonical(easybuild_version) >= convertToCanonical("5.1") or mode()
228235
setenv ("EASYBUILD_SEARCH_PATH_CPP_HEADERS", "include_paths")
229236
end
230237
end
238+
239+
-- Avoid locale related problems by (temporarily) overwriting the locale with one we know exists ('locale -a')
240+
pushenv ("LANG", "C.utf8")
241+
pushenv ("LC_ALL", "C.utf8")
242+
pushenv ("LC_CTYPE", "C.utf8")
243+
pushenv ("LC_MESSAGES", "C.utf8")
244+
231245
"""
232246

233247
moduleclass = 'devel'

‎EESSI-install-software.sh‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ else
149149

150150
# make sure the the software and modules directory exist
151151
# (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
152-
mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
152+
mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
153153

154154
# If EESSI_ACCELERATOR_TARGET_OVERRIDE is defined, we are building for an accelerator target
155155
# In that case, make sure the modulepath for the accelerator subdir exists, otherwise the EESSI module will not
@@ -160,7 +160,7 @@ else
160160
# Note that ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
161161
# is only the correct path if EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE is not set
162162
if [ -z $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE ]; then
163-
mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
163+
mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
164164
else
165165
# At runtime, one might want to use a different CPU subdir for a given accelerator. E.g. one could use
166166
# a zen2 CPU subdir on a zen4 node if the required GPU software isn't available in the zen4 tree.
@@ -368,7 +368,9 @@ else
368368
fi
369369

370370
# use PR patch file to determine in which easystack files stuff was added
371-
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing')
371+
# Note that we exclude the scripts/gpu_support/ dir, since those are not meant to be built in the
372+
# software-layer, but they are helper easystacks for installing e.g. CUDA in host_injections
373+
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | (grep -v "scripts/gpu_support/" || true))
372374
if [ -z "${changed_easystacks}" ]; then
373375
echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here
374376
else
@@ -382,44 +384,44 @@ else
382384
# make sure that easystack file being picked up is for EESSI version that we're building for...
383385
echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO})/${EESSI_VERSION}/"
384386
if [ $? -ne 0 ]; then
385-
fatal_error "Easystack file ${easystack_file} is not intended for EESSI version ${EESSI_VERSION}, giving up!"
386-
fi
387+
echo_yellow "Easystack file ${easystack_file} is not intended for EESSI version ${EESSI_VERSION}, skipping it..."
388+
else
389+
echo -e "Processing easystack file ${easystack_file}...\n\n"
390+
391+
# determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
392+
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
387393

388-
echo -e "Processing easystack file ${easystack_file}...\n\n"
394+
# load EasyBuild module (will be installed if it's not available yet)
395+
source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
389396

390-
# determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
391-
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
397+
${EB} --show-config
392398

393-
# load EasyBuild module (will be installed if it's not available yet)
394-
source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
399+
echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
395400

396-
${EB} --show-config
401+
if [ -f ${easystack_file} ]; then
402+
echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
397403

398-
echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
404+
if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
405+
${EB} --easystack ${easystack_file} --rebuild
406+
else
407+
${EB} --easystack ${easystack_file} --robot
408+
fi
409+
ec=$?
399410

400-
if [ -f ${easystack_file} ]; then
401-
echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
411+
# copy EasyBuild log file if EasyBuild exited with an error
412+
if [ ${ec} -ne 0 ]; then
413+
eb_last_log=$(unset EB_VERBOSE; eb --last-log)
414+
# copy to current working directory
415+
cp -a ${eb_last_log} .
416+
echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
417+
# copy to build logs dir (with context added)
418+
copy_build_log "${eb_last_log}" "${build_logs_dir}"
419+
fi
402420

403-
if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
404-
${EB} --easystack ${easystack_file} --rebuild
421+
$TOPDIR/check_missing_installations.sh ${easystack_file} ${pr_diff}
405422
else
406-
${EB} --easystack ${easystack_file} --robot
423+
fatal_error "Easystack file ${easystack_file} not found!"
407424
fi
408-
ec=$?
409-
410-
# copy EasyBuild log file if EasyBuild exited with an error
411-
if [ ${ec} -ne 0 ]; then
412-
eb_last_log=$(unset EB_VERBOSE; eb --last-log)
413-
# copy to current working directory
414-
cp -a ${eb_last_log} .
415-
echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
416-
# copy to build logs dir (with context added)
417-
copy_build_log "${eb_last_log}" "${build_logs_dir}"
418-
fi
419-
420-
$TOPDIR/check_missing_installations.sh ${easystack_file} ${pr_diff}
421-
else
422-
fatal_error "Easystack file ${easystack_file} not found!"
423425
fi
424426

425427
done

‎bot/check-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
503503
modules_entries=$(grep "${prefix}/modules" ${tmpfile})
504504
software_entries=$(grep "${prefix}/software" ${tmpfile})
505505
reprod_entries=$(grep "${prefix}/reprod" ${tmpfile})
506-
reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
506+
reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
507507
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod")
508508
other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
509509
modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)

0 commit comments

Comments
 (0)