replace run_cmd with run_shell_cmd in custom easyblock for Perl (perl.py)

easybuild/easyblocks/generic/cargo.py

@@ -29,10 +29,11 @@
 """
 
 import os
+import re
 
 import easybuild.tools.environment as env
 import easybuild.tools.systemtools as systemtools
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.filetools import extract_file, change_dir
@@ -102,13 +103,14 @@ def __init__(self, *args, **kwargs):
 
         # Populate sources from "crates" list of tuples (only once)
         if self.cfg['crates']:
-            # copy list of crates, so we can wipe 'crates' easyconfig parameter,
+            # Move 'crates' list from easyconfig parameter to property,
             # to avoid that creates are processed into 'sources' easyconfig parameter again
-            # when easyblock is initialized again using same parsed easyconfig
+            # when easyblock is initialized again using the same parsed easyconfig
             # (for example when check_sha256_checksums function is called, like in easyconfigs test suite)
-            self.crates = self.cfg['crates'][:]
+            self.crates = self.cfg['crates']
+            self.cfg['crates'] = []
             sources = []
-            for crate_info in self.cfg['crates']:
+            for crate_info in self.crates:
                 if len(crate_info) == 2:
                     crate, version = crate_info
                     sources.append({
@@ -119,18 +121,16 @@ def __init__(self, *args, **kwargs):
                     })
                 else:
                     crate, version, repo, rev = crate_info
-                    url, repo_name_git = repo.rsplit('/', maxsplit=1)
+                    url, repo_name = repo.rsplit('/', maxsplit=1)
+                    if repo_name.endswith('.git'):
+                        repo_name = repo_name[:-4]
                     sources.append({
-                        'git_config': {'url': url, 'repo_name': repo_name_git[:-4], 'commit': rev},
+                        'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev},
                         'filename': crate + '-' + version + '.tar.gz',
-                        'source_urls': [CRATESIO_SOURCE],
                     })
 
             self.cfg.update('sources', sources)
 
-            # set 'crates' easyconfig parameter to empty list to prevent re-processing into sources
-            self.cfg['crates'] = []
-
     def extract_step(self):
         """
         Unpack the source files and populate them with required .cargo-checksum.json if offline
@@ -242,7 +242,11 @@ def generate_crate_list(sourcedir):
     cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml'))
     cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock'))
 
-    app_name = cargo_toml['package']['name']
+    try:
+        app_name = cargo_toml['package']['name']
+    except KeyError:
+        app_name = os.path.basename(os.path.abspath(sourcedir))
+        print_warning('Did not find a [package] name= entry. Assuming it is the folder name: ' + app_name)
     deps = cargo_lock['package']
 
     app_in_cratesio = False
@@ -258,8 +262,14 @@ def generate_crate_list(sourcedir):
                 if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index':
                     crates.append((name, version))
                 else:
-                    # Lock file has revision#revision in the url for some reason.
-                    crates.append((name, version, dep['source'].rsplit('#', maxsplit=1)[0]))
+                    # Lock file has revision#revision in the url
+                    url, rev = dep['source'].rsplit('#', maxsplit=1)
+                    for prefix in ('registry+', 'git+'):
+                        if url.startswith(prefix):
+                            url = url[len(prefix):]
+                    # Remove branch name if present
+                    url = re.sub(r'\?branch=\w+$', '', url)
+                    crates.append((name, version, url, rev))
         else:
             other_crates.append((name, version))
     return app_in_cratesio, crates, other_crates
@@ -274,5 +284,5 @@ def generate_crate_list(sourcedir):
         if app_in_cratesio:
             print('    (name, version),')
         for crate_info in crates:
-            print("    ('" + "', '".join(crate_info) + "'),")
+            print("    %s," % str(crate_info))
         print(']')

easybuild/easyblocks/generic/cmakemake.py

@@ -273,6 +273,11 @@ def configure_step(self, srcdir=None, builddir=None):
             # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5
             options['CMAKE_SKIP_RPATH'] = 'ON'
 
+        # make sure that newer CMAKE picks python based on location, not just the newest python
+        # Avoids issues like e.g. https://github.com/EESSI/software-layer/pull/370#issuecomment-1785594932
+        if LooseVersion(self.cmake_version) >= '3.15':
+            options['CMAKE_POLICY_DEFAULT_CMP0094'] = 'NEW'
+
         # show what CMake is doing by default
         options['CMAKE_VERBOSE_MAKEFILE'] = 'ON'
 

easybuild/easyblocks/m/mcr.py

@@ -158,8 +158,13 @@ def set_subdir(self):
         """Determine subdirectory in installation directory"""
         # no-op is self.subdir is already set
         if self.subdir is None:
-            # determine subdirectory (e.g. v84 (2014a, 2014b), v85 (2015a), ...)
-            subdirs = glob.glob(os.path.join(self.installdir, 'v[0-9][0-9]*'))
+            # determine subdirectory
+            if LooseVersion(self.version) < LooseVersion('R2022b'):
+                # (e.g. v84 (2014a, 2014b), v85 (2015a), ...)
+                subdirs = glob.glob(os.path.join(self.installdir, 'v[0-9][0-9]*'))
+            else:
+                # (e.g. R2023a, R2023b, ...)
+                subdirs = glob.glob(os.path.join(self.installdir, 'R[0-9][0-9][0-9][0-9]*'))
             if len(subdirs) == 1:
                 self.subdir = os.path.basename(subdirs[0])
             else:

easybuild/easyblocks/o/openssl_wrapper.py

@@ -71,13 +71,16 @@ def __init__(self, *args, **kwargs):
         """Locate the installation files of OpenSSL in the host system"""
         super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
 
-        # Wrapper should have at least a major minor version numbers
-        try:
-            subversions = self.version.split('.')
-            self.majmin_version = '%s.%s' % (subversions[0], subversions[1])
-        except (AttributeError, IndexError):
-            err_msg = "Wrapper OpenSSL version does not have any subversion: %s"
-            raise EasyBuildError(err_msg, self.version)
+        # Wrapper should have at least a major minor version numbers for OpenSSL before version 3+
+        if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0:
+            self.majmin_version = self.version
+        else:
+            try:
+                subversions = self.version.split('.')
+                self.majmin_version = '%s.%s' % (subversions[0], subversions[1])
+            except (AttributeError, IndexError):
+                err_msg = "Wrapper OpenSSL version does not have any subversion: %s"
+                raise EasyBuildError(err_msg, self.version)
 
         # Set minimum OpenSSL version
         min_openssl_version = self.cfg.get('minimum_openssl_version')
@@ -116,6 +119,10 @@ def __init__(self, *args, **kwargs):
                 LINUX: ('so.3', ),
                 DARWIN: ('3.dylib', ),
             },
+            '3': {
+                LINUX: ('so.3', ),
+                DARWIN: ('3.dylib', ),
+            },
         }
 
         os_type = get_os_type()
@@ -140,6 +147,7 @@ def __init__(self, *args, **kwargs):
             '1.0': 'engines',
             '1.1': 'engines-1.1',
             '3.0': 'engines-3',
+            '3': 'engines-3',
         }
         self.target_ssl_engine = openssl_engines[self.majmin_version]
 
@@ -368,9 +376,14 @@ def sanity_check_step(self):
             'dirs': ssl_dirs,
         }
 
+        if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0:
+            ssl_ver_comp_chars = 1
+        else:
+            ssl_ver_comp_chars = 3
+
         custom_commands = [
             # make sure that version mentioned in output of 'openssl version' matches version we are using
-            "ssl_ver=$(openssl version); [ ${ssl_ver:8:3} == '%s' ]" % self.majmin_version,
+            "ssl_ver=$(openssl version); [ ${ssl_ver:8:%s} == '%s' ]" % (ssl_ver_comp_chars, self.majmin_version),
             "echo | openssl s_client -connect github.com:443 -verify 9 | grep 'Verify return code: 0 (ok)'",
         ]
 

easybuild/easyblocks/p/perl.py

@@ -40,7 +40,7 @@
 from easybuild.tools.environment import setvar, unset_env_vars
 from easybuild.tools.modules import get_software_root
 from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import run_shell_cmd
 
 # perldoc -lm seems to be the safest way to test if a module is available, based on exit code
 EXTS_FILTER_PERL_MODULES = ("perldoc -lm %(ext_name)s ", "")
@@ -117,7 +117,7 @@ def configure_step(self):
             unset_env_vars(['COLUMNS'])
 
         cmd = '%s ./Configure -de %s' % (self.cfg['preconfigopts'], configopts)
-        run_cmd(cmd, log_all=True, simple=True)
+        run_shell_cmd(cmd)
 
     def test_step(self):
         """Test Perl build via 'make test'."""
@@ -140,7 +140,7 @@ def test_step(self):
             # specify locale to be used, to avoid that a handful of tests fail
             cmd = "export LC_ALL=C && %s" % cmd
 
-            run_cmd(cmd, log_all=False, log_ok=False, simple=False)
+            run_shell_cmd(cmd, fail_on_error=False)
 
     def prepare_for_extensions(self):
         """
@@ -199,8 +199,8 @@ def get_major_perl_version():
     Returns the major verson of the perl binary in the current path
     """
     cmd = "perl -MConfig -e 'print $Config::Config{PERL_API_REVISION}'"
-    (perlmajver, _) = run_cmd(cmd, log_all=True, log_output=True, simple=False)
-    return perlmajver
+    res = run_shell_cmd(cmd)
+    return res.output
 
 
 def get_site_suffix(tag):
@@ -213,6 +213,7 @@ def get_site_suffix(tag):
     """
     perl_cmd = 'my $a = $Config::Config{"%s"}; $a =~ s/($Config::Config{"siteprefix"})//; print $a' % tag
     cmd = "perl -MConfig -e '%s'" % perl_cmd
-    (sitesuffix, _) = run_cmd(cmd, log_all=True, log_output=True, simple=False)
+    res = run_shell_cmd(cmd)
+    sitesuffix = res.output
     # obtained value usually contains leading '/', so strip it off
     return sitesuffix.lstrip(os.path.sep)

easybuild/easyblocks/p/petsc.py

@@ -257,6 +257,10 @@ def configure_step(self):
                         ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD",
                                    "COLAMD", "CXSparse", "LDL", "RBio", "SLIP_LU", "CAMD", "AMD"]
 
+                    # SLIP_LU was replaced by SPEX in SuiteSparse >= 6.0
+                    if LooseVersion(get_software_version('SuiteSparse')) >= LooseVersion("6.0"):
+                        ss_libs = [x if x != "SLIP_LU" else "SPEX" for x in ss_libs]
+
                     suitesparse_inc = os.path.join(suitesparse, "include")
                     inc_spec = "-include=[%s]" % suitesparse_inc
 
@@ -275,8 +279,9 @@ def configure_step(self):
                 self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]]))
 
             # set PETSC_DIR for configure (env) and build_step
-            env.setvar('PETSC_DIR', self.cfg['start_dir'])
-            self.cfg.update('buildopts', 'PETSC_DIR=%s' % self.cfg['start_dir'])
+            petsc_dir = self.cfg['start_dir'].rstrip(os.path.sep)
+            env.setvar('PETSC_DIR', petsc_dir)
+            self.cfg.update('buildopts', 'PETSC_DIR=%s' % petsc_dir)
 
             if self.cfg['sourceinstall']:
                 if self.petsc_arch:

easybuild/easyblocks/p/python.py

@@ -376,16 +376,19 @@ def configure_step(self):
             if tcltk_maj_min_ver != '.'.join(tkver.split('.')[:2]):
                 raise EasyBuildError("Tcl and Tk major/minor versions don't match: %s vs %s", tclver, tkver)
 
-            self.cfg.update('configopts', "--with-tcltk-includes='-I%s/include -I%s/include'" % (tcl, tk))
-
             tcl_libdir = os.path.join(tcl, get_software_libdir('Tcl'))
             tk_libdir = os.path.join(tk, get_software_libdir('Tk'))
             tcltk_libs = "-L%(tcl_libdir)s -L%(tk_libdir)s -ltcl%(maj_min_ver)s -ltk%(maj_min_ver)s" % {
                 'tcl_libdir': tcl_libdir,
                 'tk_libdir': tk_libdir,
                 'maj_min_ver': tcltk_maj_min_ver,
             }
-            self.cfg.update('configopts', "--with-tcltk-libs='%s'" % tcltk_libs)
+            if LooseVersion(self.version) < '3.11':
+                self.cfg.update('configopts', "--with-tcltk-includes='-I%s/include -I%s/include'" % (tcl, tk))
+                self.cfg.update('configopts', "--with-tcltk-libs='%s'" % tcltk_libs)
+            else:
+                env.setvar('TCLTK_CFLAGS', '-I%s/include -I%s/include' % (tcl, tk))
+                env.setvar('TCLTK_LIBS', tcltk_libs)
 
         # don't add user site directory to sys.path (equivalent to python -s)
         # This matters e.g. when python installs the bundled pip & setuptools (for >= 3.4)

easybuild/easyblocks/r/r.py

@@ -70,8 +70,11 @@ def configure_step(self):
         for dep in ['Tcl', 'Tk']:
             root = get_software_root(dep)
             if root:
-                dep_config = os.path.join(root, 'lib', '%sConfig.sh' % dep.lower())
-                self.cfg.update('configopts', '--with-%s-config=%s' % (dep.lower(), dep_config))
+                for libdir in ['lib', 'lib64']:
+                    dep_config = os.path.join(root, libdir, '%sConfig.sh' % dep.lower())
+                    if os.path.exists(dep_config):
+                        self.cfg.update('configopts', '--with-%s-config=%s' % (dep.lower(), dep_config))
+                        break
 
         if "--with-x=" not in self.cfg['configopts'].lower():
             if get_software_root('X11'):

easybuild/easyblocks/v/vep.py

@@ -30,6 +30,7 @@
 import easybuild.tools.environment as env
 from easybuild.easyblocks.perl import get_major_perl_version
 from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import print_warning
 from easybuild.tools.filetools import apply_regex_substitutions
 from easybuild.tools.modules import get_software_version, get_software_root
@@ -48,6 +49,15 @@ def __init__(self, *args, **kwargs):
 
         self.api_mods_subdir = os.path.join('modules', 'api')
 
+    @staticmethod
+    def extra_options(extra_vars=None):
+        """Extra easyconfig parameters specific to VEP easyblock."""
+        extra_vars = EasyBlock.extra_options(extra_vars)
+        extra_vars.update({
+            'species': ['all', "Comma-separated list of species to pass to INSTALL.pl", CUSTOM],
+        })
+        return extra_vars
+
     def configure_step(self):
         """No custom configuration procedure for VEP."""
         pass
@@ -97,8 +107,8 @@ def install_step(self):
             # l: Bio::DB::HTS, should be provided via EasyBuild
             # p: plugins
             '--AUTO af',
-            # install all species
-            '--SPECIES all',
+            # install selected species
+            '--SPECIES %s' % self.cfg['species'],
             # don't update VEP during installation
             '--NO_UPDATE',
             # location to install Perl API modules into