Merge branch 'main' of github.com:EESSI/software-layer-scripts into zstd_tarballs

Author: bedroge

EESSI-extend-easybuild.eb

@@ -70,7 +70,7 @@ modextravars = {
 # EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}
 # EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH}
 #
-# And also some optional ones based on the kind of installation
+# And also some optional ones based on the installation mode
 # EASYBUILD_SET_GID_BIT
 # EASYBUILD_GROUP_WRITABLE_INSTALLDIR
 # EASYBUILD_UMASK
@@ -212,7 +212,11 @@ easybuild_version = os.getenv("EBVERSIONEASYBUILD") or easybuild_version
 eessi_version = os.getenv("EESSI_VERSION") or "2023.06"
 
 -- Set environment variables that are EasyBuild version specific
-if convertToCanonical(easybuild_version) > convertToCanonical("4") then
+-- Do unload unconditionally, so that even if EB versions were switched in the meantime, this gets unset
+-- This avoids issues where EESSI-extend is first loaded with EB => 5.1 (which set these vars)
+-- but then EB is swapped for a version < 5.1 and then EESSI-extend is unloaded (which would not unset
+-- these vars if we did it conditional on the EB version)
+if convertToCanonical(easybuild_version) >= convertToCanonical("5.1") or mode() == "unload" then
   setenv ("EASYBUILD_STRICT_RPATH_SANITY_CHECK", "1")
   setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
   setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")

EESSI-install-software.sh

@@ -150,6 +150,29 @@ else
       # make sure the the software and modules directory exist
       # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
       mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
+
+      # If EESSI_ACCELERATOR_TARGET_OVERRIDE is defined, we are building for an accelerator target
+      # In that case, make sure the modulepath for the accelerator subdir exists, otherwise the EESSI module will not
+      # set EESSI_ACCELERATOR_TARGET and the if-condition later in this script which checks if EESSI_ACCELERATOR_TARGET
+      # is equal to EESSI_ACCELERATOR_TARGET_OVERRIDE will fail 
+      # See https://github.com/EESSI/software-layer-scripts/pull/59#issuecomment-3173593882
+      if [ -n $EESSI_ACCELERATOR_TARGET_OVERRIDE ]; then
+          # Note that ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
+          # is only the correct path if EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE is not set
+          if [ -z $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE ]; then
+              mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
+          else
+              # At runtime, one might want to use a different CPU subdir for a given accelerator. E.g. one could use
+              # a zen2 CPU subdir on a zen4 node if the required GPU software isn't available in the zen4 tree.
+              # At build time, this doesn't make a lot of sense: we'd probably build in a CPU prefix that is different
+              # from what the code will be optimized for, and we wouldn't want that
+              # So this message _should_ never be printed...
+              msg="When building the software subdirectory for the CPU should almost certainly be that of the host."
+              msg="$msg If you think this is incorrect, please implement behaviour that makes sense in "
+              msg="$msg EESSI-software-installation.sh, essentially replacing this error."
+              fatal_error "$msg"
+          fi
+      fi
   )
 fi
 
@@ -294,6 +317,7 @@ source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION}
 echo "DEBUG: after loading EESSI-extend //  EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
 
 # Install full CUDA SDK and cu* libraries in host_injections
+# (This is done *before* configuring EasyBuild as it may rely on an older EB version)
 # Hardcode this for now, see if it works
 # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
 # Allow skipping CUDA SDK install in e.g. CI environments
@@ -315,6 +339,7 @@ if nvidia_gpu_available; then
     ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 fi
 
+
 if [ ! -z "${shared_fs_path}" ]; then
     shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
     echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
@@ -323,7 +348,7 @@ fi
 
 # if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available
 if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then
-    CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all
+    CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all
     if [ -d ${CPU_ONLY_MODULES_PATH} ]; then
         module use ${CPU_ONLY_MODULES_PATH}
     else
@@ -414,7 +439,7 @@ lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua"
 echo "DEBUG: lmod_rc_file='${lmod_rc_file}'"
 if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
     # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file
-    lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@")
+    lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/${EESSI_ACCELERATOR_TARGET}@@")
     echo "Path to lmodrc.lua changed to '${lmod_rc_file}'"
 fi
 lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?)
@@ -427,7 +452,7 @@ fi
 lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua"
 if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
     # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file
-    lmod_sitepackage_file=$(echo ${lmod_sitepackage_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@")
+    lmod_sitepackage_file=$(echo ${lmod_sitepackage_file} | sed "s@/${EESSI_ACCELERATOR_TARGET}@@")
     echo "Path to SitePackage.lua changed to '${lmod_sitepackage_file}'"
 fi
 sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?)

bot/build.sh

@@ -267,11 +267,15 @@ TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 timestamp=$(date +%s)
 # to set EESSI_VERSION we need to source init/eessi_defaults now
 source $software_layer_dir/init/eessi_defaults
-# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we 
+# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we 
 # append the project (subdirectory) name to the end tarball name. This is information
 # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
 # appended
-export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
+    export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+else
+    export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+fi
 
 # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
 echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'"

create_lmodsitepackage.py

@@ -123,13 +123,32 @@
     local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/site_specific_config/gpu/.\\n"
     if packagesList[simpleName] then
         -- simpleName is a module in packagesList
-        -- get the full host_injections path
+        -- first, check the old host_injections path prior to https://github.com/EESSI/software-layer-scripts/pull/59
+        -- If that exists, print a more targetted, explanatory warning
+        local previousHostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
+        local previousPackageEasyBuildDir = previousHostInjections .. "/software/" .. t.modFullName .. "/easybuild"
+        local previousPackageDirExists = isDir(previousPackageEasyBuildDir)
+
+        -- get the host_injections path, and add only the EESSI_CPU_FAMILY at the end
+        local strip_suffix = os.getenv('EESSI_VERSION') .. "/software/" .. os.getenv('EESSI_OS_TYPE') .. "/"
+        strip_suffix = strip_suffix .. os.getenv('EESSI_SOFTWARE_SUBDIR')
         local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
+        hostInjections = string.gsub(hostInjections, strip_suffix, os.getenv('EESSI_CPU_FAMILY'))
 
         -- build final path where the software should be installed
         local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
         local packageDirExists = isDir(packageEasyBuildDir)
-        if not packageDirExists then
+        if previousPackageDirExists and not packageDirExists then
+            local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI "
+            advice = advice .. "due to licencing. You will need to install a full copy of the " .. simpleName .. " package where EESSI "
+            advice = advice .. "can find it.\\n"
+            advice = advice .. "Note that a full copy is installed at " .. previousHostInjections .. "/software/" .. t.modFullName .. ". "
+            advice = advice .. "However, EESSI expects it in a different location since Aug'25, namely at "
+            advice = advice .. hostInjections .. "/software/" .. t.modFullName .. ". "
+            advice = advice .. "Please re-install the package at the new location. "
+            advice = advice .. refer_to_docs
+            LmodError("\\nYou requested to load ", simpleName, " ", advice)
+        elseif not packageDirExists then
             local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI "
             advice = advice .. "due to licencing. You will need to install a full copy of the " .. simpleName .. " package where EESSI "
             advice = advice .. "can find it.\\n"
@@ -293,7 +312,7 @@ def error(msg):
 # the install path (if it exists)
 accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET")
 if accel_subdir:
-    sitepackage_path = sitepackage_path.replace("/accel/%s" % accel_subdir, '')
+    sitepackage_path = sitepackage_path.replace("/%s" % accel_subdir, '')
 try:
     os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True)
     with open(sitepackage_path, 'w') as fp:

easystacks/README.md

@@ -0,0 +1,5 @@
+WARNING: in principle _all_ easystack files should go into EESSI/software-layer, not in EESSI/software-layer-scripts. Easystack files are only added in EESSI/software-layer-scripts by exception, for example when the (re)deployment of the software has to be done synchronously with a change in EESSI/software-layer-scripts.
+
+Here, we list past deployments for which this was the case (and why):
+
+[PR#59](https://github.com/EESSI/software-layer-scripts/pull/59): modified the prefix in which `install_cuda_and_libraries.sh` installs the CUDA toolkit within `host_injections`. Also, updated the Lmod SitePackage.lua to print an informative message in case the CUDA Toolkit is found in the old location. This requires synchronous deployment of new CUDA and cuDNN installations in the software layer, because the symlinks from these installations should be redirected to the new prefix in `host_injections`.

easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml

@@ -0,0 +1,13 @@
+# In https://github.com/EESSI/software-layer-scripts/pull/59 we introduced a new location for
+# installing the CUDA toolkit within the host_injections directory. This requires reinstallation
+# of CUDA and cuDNN to make sure all symlinks point to these new locations
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: CUDA
+  - CUDA-12.4.0.eb:
+      options:
+        accept-eula-for: CUDA
+  - cuDNN-8.9.2.26-CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: cuDNN