Merge branch 'main' into fix-llvm

Author: bedroge

EESSI-extend-easybuild.eb

@@ -96,12 +96,24 @@ sysroot = os.getenv("EESSI_EPREFIX")
 -- Check if we have GPU capabilities and configure CUDA compute capabilities
 eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET")
 if (eessi_accelerator_target ~= nil) then
-  cuda_compute_capability = string.match(eessi_accelerator_target, "^accel/nvidia/cc([0-9][0-9])$")
+  cuda_compute_capability = string.match(eessi_accelerator_target, "^accel/nvidia/cc([0-9]+)$")
   if (cuda_compute_capability ~= nil) then
-    easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) ..  "." .. cuda_compute_capability:sub(2, 2)
+    -- The last digit should be the minor version, insert a dot in the one-but-last position
+    major_version = cuda_compute_capability:sub(1, #cuda_compute_capability - 1)
+    minor_version = cuda_compute_capability:sub(#cuda_compute_capability)
+    easybuild_cuda_compute_capabilities = string.format("%s.%s", major_version, minor_version)
   else
     LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target)
   end
+
+  -- If architectures are 9.0, 10.0 or 12.0, enable architecture or family-specific optimizations
+  if easybuild_cuda_compute_capabilities == '9.0' then
+    easybuild_cuda_compute_capabilities = '9.0a'
+  elseif easybuild_cuda_compute_capabilities == '10.0' then
+    easybuild_cuda_compute_capabilities = '10.0f'
+  elseif easybuild_cuda_compute_capabilities == '12.0' then
+    easybuild_cuda_compute_capabilities = '12.0f'
+  end
 end
 
 -- Some environment variables affect behaviour, let's gather them once
@@ -228,6 +240,7 @@ if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuil
   setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
   setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")
   setenv ("EASYBUILD_LOCAL_VAR_NAMING_CHECK", "error")
+  setenv ("EASYBUILD_PARALLEL_EXTENSIONS_INSTALL", "1")
   -- Set environment variables that are EESSI version specific
   if convertToCanonical(eessi_version) > convertToCanonical("2023.06") then
     setenv ("EASYBUILD_PREFER_PYTHON_SEARCH_PATH", "EBPYTHONPREFIXES")

EESSI-install-software.sh

@@ -149,7 +149,7 @@ else
 
       # make sure the the software and modules directory exist
       # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
-      mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
+      mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
 
       # If EESSI_ACCELERATOR_TARGET_OVERRIDE is defined, we are building for an accelerator target
       # In that case, make sure the modulepath for the accelerator subdir exists, otherwise the EESSI module will not
@@ -160,7 +160,7 @@ else
           # Note that ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
           # is only the correct path if EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE is not set
           if [ -z $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE ]; then
-              mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
+              mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
           else
               # At runtime, one might want to use a different CPU subdir for a given accelerator. E.g. one could use
               # a zen2 CPU subdir on a zen4 node if the required GPU software isn't available in the zen4 tree.
@@ -207,7 +207,7 @@ fi
 
 # the install_scripts.sh script relies on knowing the location of the PR diff
 # assume there's only one diff file that corresponds to the PR patch file
-pr_diff=$(ls [0-9]*.diff | head -1)
+pr_diff=$(ls [0-9]*.diff | head -n 1)
 export PR_DIFF="$PWD/$pr_diff"
 
 # Only run install_scripts.sh if not in dev.eessi.io for security
@@ -249,6 +249,8 @@ fi
 
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+# force Python's stdout and stderr streams to be unbuffered
+export PYTHONUNBUFFERED=1
 
 # if we run the script for the first time, e.g., to start building for a new
 #   stack, we need to ensure certain files are present in
@@ -368,7 +370,9 @@ else
 fi
 
 # use PR patch file to determine in which easystack files stuff was added
-changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing') 
+# Note that we exclude the scripts/gpu_support/ dir, since those are not meant to be built in the
+# software-layer, but they are helper easystacks for installing e.g. CUDA in host_injections
+changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | (grep -v "scripts/gpu_support/" || true)) 
 if [ -z "${changed_easystacks}" ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else

bot/build.sh

@@ -171,6 +171,22 @@ else
 fi
 echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'"
 
+# Log the full lscpu and os-release info:
+lscpu > _bot_job${SLURM_JOB_ID}.lscpu
+cat /etc/os-release > _bot_job${SLURM_JOB_ID}.os
+
+# Also: fetch CPU flags into an array, so that we can implement a hard check against a reference
+lscpu_flags_line=$(lscpu | grep "Flags:" || echo "")
+# strip leading "Flags:" and spaces, and put result in a bash array
+if [[ $lscpu_flags =~ Flags:\ (.*) ]]; then lscpu_flags=(${BASH_REMATCH[1]}); fi
+# for now, just print
+echo "bot/build.sh: CPU flags=${lscpu_flags[@]}"
+# TODO: an actual comparison with a reference bash array, e.g. through
+# diff_result=$(diff <(printf "%s\n" "${lscpu_flags[@]}" | sort) <(printf "%s\n" "${lscpu_flags_ref[@]}" | sort))
+# if [ ! -z "$diff_result" ]; then
+#    echo "bot/build.sh: ERROR: difference between reported lscpu flags and reference for this ($EESSI_SOFTWARE_SUBDIR_OVERRIDE) CPU architecture. This could mean an incorrect build host was used to build for this target.
+# fi
+
 # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
 EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
@@ -265,16 +281,24 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 timestamp=$(date +%s)
+# determine compression/extension for tarball, check in order of preference
+if [[ -x "$(command -v zstd)" ]]; then
+    tarball_extension="tar.zst"
+elif [[ -x "$(command -v gzip)" ]]; then
+    tarball_extension="tar.gz"
+else
+    tarball_extension="tar"
+fi
 # to set EESSI_VERSION we need to source init/eessi_defaults now
 source $software_layer_dir/init/eessi_defaults
 # Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we 
 # append the project (subdirectory) name to the end tarball name. This is information
 # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
 # appended
 if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
-    export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+    export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
 else
-    export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+    export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
 fi
 
 # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
@@ -288,8 +312,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
 echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}"
 $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

bot/check-build.sh

@@ -17,7 +17,7 @@
 
 #  - SUCCESS (all of)
 #    - working directory contains slurm-JOBID.out file
-#    - working directory contains eessi*tar.gz
+#    - working directory contains eessi*tar*
 #    - no message FATAL
 #    - no message ERROR
 #    - no message FAILED
@@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
 fi
 
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    TGZ=-1
+    TARBALL_CREATED=-1
     TARBALL=
     if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-      GP_tgz_created="\.tar\.gz created!"
-      grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
+      GP_tarball_created="\.tar.* created!"
+      grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u)
       if [[ $? -eq 0 ]]; then
-          TGZ=1
+          TARBALL_CREATED=1
           TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
       else
-          TGZ=0
+          TARBALL_CREATED=0
       fi
       # have to be careful to not add searched for pattern into slurm out file
-      [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+      [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'"
       [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
     fi
 fi
@@ -190,7 +190,7 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+    [[ ${VERBOSE} -ne 0 ]] && echo "  TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 fi
 
 # Here, we try to do some additional analysis on the output file
@@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
    [[ ${NO_MISSING} -eq 1 ]] && \
-   [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
+   [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \
    [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
     # SUCCESS
     status="SUCCESS"
@@ -429,9 +429,9 @@ failure_msg="no message matching <code>${GP_no_missing}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    success_msg="found message matching <code>${GP_tgz_created}</code>"
-    failure_msg="no message matching <code>${GP_tgz_created}</code>"
-    comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+    success_msg="found message matching <code>${GP_tarball_created}</code>"
+    failure_msg="no message matching <code>${GP_tarball_created}</code>"
+    comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}")
 fi
 
 # Now, do the actual replacement of __DETAILS_FMT__
@@ -478,7 +478,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
         size="$(stat --dereference --printf=%s ${TARBALL})"
         size_mib=$((${size} >> 20))
         tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
-        tar tf ${TARBALL} > ${tmpfile}
+        if [[ "${TARBALL}" == *.tar.zst ]]; then
+          tar --use-compress-program=zstd -tf ${TARBALL} > ${tmpfile}
+        elif [[ "${TARBALL}" == *.tar.gz ]]; then
+          tar --use-compress-program=gzip -tf ${TARBALL} > ${tmpfile}
+        elif [[ "${TARBALL}" == *.tar ]]; then
+          tar -tf ${TARBALL} > ${tmpfile}
+        else
+          echo "ERROR: Unsupported tarball extension!" >&2
+          exit 1
+        fi
         entries=$(cat ${tmpfile} | wc -l)
         # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
         # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
@@ -503,7 +512,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
         modules_entries=$(grep "${prefix}/modules" ${tmpfile})
         software_entries=$(grep "${prefix}/software" ${tmpfile})
         reprod_entries=$(grep "${prefix}/reprod" ${tmpfile})
-        reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
+        reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
         other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod")
         other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
         modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)

bot/test.sh

@@ -100,7 +100,7 @@ if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
 fi
 
 # try to determine tmp directory from build job
-RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
+RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -n 1 | awk '{print $2}')
 
 if [[ -z ${RESUME_DIR} ]]; then
   RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)
@@ -189,6 +189,10 @@ EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
 
+# Get node_type from .architecture.node_type in ${JOB_CFG_FILE}
+export BOT_NODE_TYPE=$(cfg_get_value "architecture" "node_type")
+echo "bot/test.sh: BOT_NODE_TYPE='${BOT_NODE_TYPE}"
+
 # prepare arguments to eessi_container.sh common to build and tarball steps
 declare -a COMMON_ARGS=()
 COMMON_ARGS+=("--verbose")
@@ -237,6 +241,9 @@ fi
 if [[ ${SHARED_FS_PATH} ]]; then
     TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 fi
+if [[ ${BOT_NODE_TYPE} ]]; then
+    TEST_SUITE_ARGS+=("--partition" "${BOT_NODE_TYPE}")
+fi
 # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 

create_tarball.sh

@@ -12,7 +12,7 @@ eessi_tmpdir=$1
 eessi_version=$2
 cpu_arch_subdir=$3
 accel_subdir=$4
-target_tgz=$5
+target_tarball=$5
 
 tmpdir=`mktemp -d`
 echo ">> tmpdir: $tmpdir"
@@ -114,10 +114,9 @@ fi
 
 topdir=${cvmfs_repo}/versions/
 
-echo ">> Creating tarball ${target_tgz} from ${topdir}..."
-tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list}
-
-echo ${target_tgz} created!
+echo ">> Creating tarball ${target_tarball} from ${topdir}..."
+tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list}
+echo ${target_tarball} created!
 
 echo ">> Cleaning up tmpdir ${tmpdir}..."
 rm -r ${tmpdir}