Merge pull request #667 from TopRichard/eessi-2023.06-EESSI_Module_bash

EESSI bash initialization to module file

.github/workflows/tests_eessi_module.yml

@@ -0,0 +1,85 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_module_functionality in software.eessi.io
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+          - 2023.06        
+    steps:
+      - name: Check out software-layer repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Mount EESSI CernVM-FS pilot repository
+        uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+        with:
+          cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+          cvmfs_http_proxy: DIRECT
+          cvmfs_repositories: software.eessi.io
+
+      - name: Test for making sure spider cache is being used and not being rebuilt
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          configfile="configfile.txt"
+          module -T load EESSI/${{matrix.EESSI_VERSION}}
+          module --config > "${configfile}" 2>&1
+          grep cache "${configfile}" | grep software | grep -v compat
+          if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then
+              echo "EESSI spider cache is being used"
+          else
+              echo "EESSI spider cache is being rebuilt" >&2
+              exit 1
+          fi
+          env | grep LMOD
+          module purge
+          unset MODULEPATH
+    
+      - name: Test for archdetect_cpu functionality with invalid path
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          set +e  # Do not exit immediately if a command exits with a non-zero status          
+          export EESSI_ARCHDETECT_OPTIONS="dummy/cpu"
+          outfile="outfile.txt"
+          module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1
+          cat "${outfile}"
+          if grep -q  "Software directory check" "${outfile}"; then
+              echo "Test for picking up invalid path on \${archdetect_cpu} PASSED"
+          else
+              echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2
+              exit 1
+          fi
+          unset EESSI_ARCHDETECT_OPTIONS
+          set -e  # Re-enable exit on non-zero status
+              
+      - name: Test for expected variables while adding dummy cpu archs and loading EESSI module 
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath)
+          export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1"          
+          moduleoutfile="moduleout.txt"
+          sourceoutfile="sourceout.txt"
+          module load EESSI/${{matrix.EESSI_VERSION}}
+          env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}"
+          module unload EESSI/${{matrix.EESSI_VERSION}}
+          source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+          env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}"
+          cat "${moduleoutfile}"
+          cat "${sourceoutfile}"
+          if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then
+              echo "Test for checking env variables PASSED"
+          else 
+              echo "Test for checking env variables FAILED"  >&2
+              exit 1
+          fi
+

init/modules/EESSI/2023.06.lua

@@ -0,0 +1,70 @@
+help([[
+Description
+===========
+The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community.The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure. 
+
+More information
+================
+ - URL: https://www.eessi.io/docs/
+]])
+whatis("Description: The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community. The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.")
+whatis("URL: https://www.eessi.io/docs/")
+conflict("EESSI")
+local eessi_version = myModuleVersion()
+local eessi_repo = "/cvmfs/software.eessi.io"
+local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version)
+local eessi_os_type = "linux"
+setenv("EESSI_VERSION", eessi_version)
+setenv("EESSI_CVMFS_REPO", eessi_repo)
+setenv("EESSI_OS_TYPE", eessi_os_type)
+function archdetect_cpu()
+    local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh')
+    if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then
+        if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then
+            LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic")
+        end
+        source_sh("bash", script)
+    end
+    local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or ""
+    for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do
+        if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then
+            -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, 
+            -- since optimized software installations for Zen4 are a work-in-progress, 
+            -- see https://gitlab.com/eessi/support/-/issues/37 
+            if archdetect_filter_cpu == "x86_64/amd/zen4" then
+                archdetect_filter_cpu = "x86_64/amd/zen3"
+                if mode() == "load" then
+                    LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.")
+                end
+            end
+            return archdetect_filter_cpu
+        end
+    end
+    LmodError("Software directory check for the detected architecture failed")
+end
+local archdetect = archdetect_cpu()
+local eessi_cpu_family = archdetect:match("([^/]+)")
+local eessi_software_subdir = archdetect
+local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family)
+local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir)
+local eessi_module_path = pathJoin(eessi_software_path, "modules", "all")
+local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections")
+setenv("EPREFIX",  eessi_eprefix)
+setenv("EESSI_CPU_FAMILY", eessi_cpu_family)
+setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path)
+setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir)
+setenv("EESSI_PREFIX", eessi_prefix)
+setenv("EESSI_EPREFIX", eessi_eprefix)
+prepend_path("PATH", pathJoin(eessi_eprefix, "bin"))
+prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin"))
+setenv("EESSI_SOFTWARE_PATH", eessi_software_path)
+setenv("EESSI_MODULEPATH", eessi_module_path)
+if ( mode() ~= "spider" ) then
+    prepend_path("MODULEPATH", eessi_module_path)
+end
+prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua"))
+prepend_path("MODULEPATH", eessi_site_module_path)
+setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod"))
+if mode() == "load" then
+    LmodMessage("EESSI/" .. eessi_version .. " loaded successfully")
+end

install_scripts.sh

@@ -102,6 +102,12 @@ mc_files=(
 )
 copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Castle "${mc_files[@]}"
 
+# Copy for init/modules/EESSI directory
+mc_files=(
+   2023.06.lua
+)
+copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}"
+
 # Copy for the scripts directory
 script_files=(
     utils.sh