Adjust scalar/vector math function handling

CMake/CheckSincos.cmake

@@ -1,7 +1,6 @@
-include(${CMAKE_ROOT}/Modules/CheckCXXSourceCompiles.cmake)
+include(CheckCXXSourceCompiles)
 
 set(CMAKE_REQUIRED_LIBRARIES Math::scalar_vector_functions)
-
 set(SINCOS_TEST_SRC
     "#include \"${SINCOS_INCLUDE}\"
 int main(void) {
@@ -12,3 +11,4 @@ sincos(input,&outsin, &outcos);
 ")
 
 check_cxx_source_compiles("${SINCOS_TEST_SRC}" HAVE_SINCOS)
+unset(CMAKE_REQUIRED_LIBRARIES)

CMake/FindAMDLibM.cmake

@@ -0,0 +1,55 @@
+# Output: HAVE_AMD_LIBM
+
+message(STATUS "Setting up AMD LibM libraries")
+message(WARNING "In limited benchmarks, AMD LibM slows down QMCPACK. "
+        "Please check carefully before using it in production runs.")
+
+include(CheckCXXSourceCompiles)
+
+find_path(
+  AMD_LIBM_INCLUDE_DIR NAME amdlibm.h
+  HINTS ${AMD_LIBM_ROOT} ${AOCL_ROOT}
+  PATH_SUFFIXES include)
+
+if(NOT AMD_LIBM_INCLUDE_DIR)
+  message(FATAL_ERROR "AMD_LIBM_INCLUDE_DIR not set. Header file amdlib.h not found!")
+else()
+  message(STATUS "Header file amdlib.h found at  ${AMD_LIBM_INCLUDE_DIR}")
+endif()
+
+find_library(
+  AMD_LIBM_LIBRARY NAME amdlibm
+  HINTS ${AMD_LIBM_ROOT} ${AOCL_ROOT}
+  PATH_SUFFIXES lib)
+
+if(NOT AMD_LIBM_LIBRARY)
+  message(FATAL_ERROR "AMD_LIBM_LIBRARY not set. Library file amdlibm not found!")
+else()
+  message(STATUS "Library file amdlibm found as ${AMD_LIBM_LIBRARY}")
+endif()
+
+set(CMAKE_REQUIRED_INCLUDES "${AMD_LIBM_INCLUDE_DIR}")
+set(CMAKE_REQUIRED_LIBRARIES "${AMD_LIBM_LIBRARY}")
+check_cxx_source_compiles(
+  "
+#include <amdlibm.h>
+int main() {
+  double d_in(0), d_sin, d_cos;
+  amd_sincos(d_in, &d_sin, &d_cos);
+  float s_in(0), s_sin, s_cos;
+  amd_sincosf(s_in, &s_sin, &s_cos);
+  return 0;
+}
+"
+  HAVE_AMD_LIBM)
+unset(CMAKE_REQUIRED_INCLUDES)
+unset(CMAKE_REQUIRED_LIBRARIES)
+
+if(HAVE_AMD_LIBM)
+  target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_AMD_LIBM")
+  target_include_directories(Math::scalar_vector_functions INTERFACE "${AMD_LIBM_INCLUDE_DIR}")
+  target_link_libraries(Math::scalar_vector_functions INTERFACE "${AMD_LIBM_LIBRARY}")
+  message(STATUS "AMD LIBM found")
+else()
+  message(FATAL_ERROR "AMD LIBM not found")
+endif()

CMake/FindFFTW.cmake

@@ -20,12 +20,9 @@ if(FFTW_INCLUDE_DIR AND FFTW_LIBRARIES)
   message(STATUS "FFTW_INCLUDE_DIR=${FFTW_INCLUDE_DIR}")
   message(STATUS "FFTW_LIBRARIES=${FFTW_LIBRARIES}")
   set(FFTW_FOUND TRUE)
-  #create FFTW3 target
-  add_library(Math::FFTW3 INTERFACE IMPORTED)
-  set_target_properties(
-    Math::FFTW3
-    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIR}" INTERFACE_COMPILE_DEFINITIONS "HAVE_LIBFFTW"
-               INTERFACE_LINK_LIBRARIES "${FFTW_LIBRARIES}")
+  target_include_directories(Math::FFTW3 INTERFACE "${FFTW_INCLUDE_DIR}")
+  target_compile_definitions(Math::FFTW3 INTERFACE "HAVE_LIBFFTW")
+  target_link_libraries(Math::FFTW3 INTERFACE "${FFTW_LIBRARIES}")
 endif()
 
 mark_as_advanced(FFTW_INCLUDE_DIR FFTW_LIBRARIES FFTW_FOUND)

CMake/FindIBMMASS.cmake

@@ -81,23 +81,18 @@ if(HAVE_MASS OR HAVE_MASSV)
   set(MASS_FOUND TRUE)
   message(STATUS "MASS found: HAVE_MASS=${HAVE_MASS}, HAVE_MASSV=${HAVE_MASSV}")
 
-  #create scalar_vector_functions target
-  add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
-  set_target_properties(
-    Math::scalar_vector_functions PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MASS_INCLUDE_DIRECTORIES}"
-                                             INTERFACE_LINK_OPTIONS "${MASS_LINKER_FLAGS}")
+  target_include_directories(Math::scalar_vector_functions INTERFACE "${MASS_INCLUDE_DIRECTORIES}")
+  target_link_options(Math::scalar_vector_functions INTERFACE "${MASS_LINKER_FLAGS}")
+
   if(HAVE_MASS)
-    set_property(
-      TARGET Math::scalar_vector_functions
-      APPEND
-      PROPERTY INTERFACE_COMPILE_DEFINITIONS "HAVE_MASS")
-    set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MASS_LIBRARY}")
+    target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MASS")
+    target_link_libraries(Math::scalar_vector_functions INTERFACE "${MASS_LIBRARY}")
     set(SINCOS_INCLUDE mass.h)
   endif()
 
   if(HAVE_MASSV)
-    set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS "HAVE_MASSV")
-    set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MASSV_LIBRARY}")
+    target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MASSV")
+    target_link_libraries(Math::scalar_vector_functions INTERFACE "${MASSV_LIBRARY}")
   endif()
 
 else()

CMake/FindMKL.cmake

@@ -6,20 +6,20 @@ set(MKL_LIBRARIES ${LAPACK_LINKER_FLAGS} ${LAPACK_LIBRARIES})
 
 message(STATUS "Looking for Intel MKL library header files")
 
-# Finding and setting the MKL_INCLUDE_DIRECTORIES based on MKL_ROOT, $ENV{MKLROOT}, $ENV{MKL_ROOT}, $ENV{MKL_HOME}
+# Finding and setting the MKL_INCLUDE based on MKL_ROOT, $ENV{MKLROOT}, $ENV{MKL_ROOT}, $ENV{MKL_HOME}
 # Extremely Basic Support of common mkl module environment variables
 find_path(
-  MKL_INCLUDE_DIRECTORIES "mkl.h"
+  MKL_INCLUDE "mkl.h"
   HINTS ${MKL_ROOT} $ENV{MKLROOT} $ENV{MKL_ROOT} $ENV{MKL_HOME}
   PATH_SUFFIXES include)
-if(NOT MKL_INCLUDE_DIRECTORIES)
+if(NOT MKL_INCLUDE)
   # Finding MKL headers in the system
-  find_path(MKL_INCLUDE_DIRECTORIES "mkl.h" PATH_SUFFIXES mkl)
+  find_path(MKL_INCLUDE "mkl.h" PATH_SUFFIXES mkl)
 endif()
 
-if(MKL_INCLUDE_DIRECTORIES)
-  message(STATUS "MKL_INCLUDE_DIRECTORIES: ${MKL_INCLUDE_DIRECTORIES}")
-else(MKL_INCLUDE_DIRECTORIES)
+if(MKL_INCLUDE)
+  message(STATUS "MKL_INCLUDE: ${MKL_INCLUDE}")
+else(MKL_INCLUDE)
   message(STATUS "mkl.h cannot be found")
   if(COMPILER MATCHES "Intel")
     message(
@@ -30,32 +30,24 @@ else(MKL_INCLUDE_DIRECTORIES)
   else(COMPILER MATCHES "Intel")
     message(FATAL_ERROR "Pass mkl root directory to cmake via MKL_ROOT.")
   endif(COMPILER MATCHES "Intel")
-endif(MKL_INCLUDE_DIRECTORIES)
+endif(MKL_INCLUDE)
 
 # Check for mkl.h
 file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx"
      "#include <iostream>\n #include <mkl.h>\n int main() { return 0; }\n")
 try_compile(
   HAVE_MKL ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx
-  CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
+  CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE} "
   LINK_LIBRARIES "${MKL_LIBRARIES}"
   OUTPUT_VARIABLE MKL_OUT)
 if(NOT HAVE_MKL)
   message("${MKL_OUT}")
 endif(NOT HAVE_MKL)
 
-# Check for mkl_vml_functions.h
-file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx"
-     "#include <iostream>\n #include <mkl_vml_functions.h>\n int main() { return 0; }\n")
-try_compile(
-  HAVE_MKL_VML ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx
-  CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
-  OUTPUT_VARIABLE MKL_OUT)
-
 # Check for fftw3
 find_path(
   MKL_FFTW3 "fftw3.h"
-  HINTS ${MKL_INCLUDE_DIRECTORIES}
+  HINTS ${MKL_INCLUDE}
   PATH_SUFFIXES fftw)
 if(MKL_FFTW3)
   file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx"
@@ -71,29 +63,16 @@ endif(MKL_FFTW3)
 
 if(HAVE_MKL)
   set(MKL_FOUND TRUE)
-  message(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_VML=${HAVE_MKL_VML}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")
+  message(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")
 
   #Add BLAS_LAPACK header
-  set_target_properties(Math::BLAS_LAPACK PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL"
-                                                     INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIRECTORIES}")
-
-  if(HAVE_MKL_VML)
-    #create scalar_vector_functions target
-    add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
-    set_target_properties(
-      Math::scalar_vector_functions
-      PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL;HAVE_MKL_VML" INTERFACE_INCLUDE_DIRECTORIES
-                                                                       "${MKL_INCLUDE_DIRECTORIES}"
-                 INTERFACE_LINK_LIBRARIES "${MKL_LIBRARIES}")
-  endif(HAVE_MKL_VML)
+  target_compile_definitions(Math::BLAS_LAPACK INTERFACE "HAVE_MKL")
+  target_include_directories(Math::BLAS_LAPACK INTERFACE "${MKL_INCLUDE}")
 
   if(HAVE_MKL_FFTW3)
-    #create FFTW3 target
-    add_library(Math::FFTW3 INTERFACE IMPORTED)
-    set_target_properties(
-      Math::FFTW3
-      PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MKL_FFTW3}" INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL;HAVE_LIBFFTW"
-                 INTERFACE_LINK_LIBRARIES "${MKL_LIBRARIES}")
+    target_compile_definitions(Math::FFTW3 INTERFACE "HAVE_MKL;HAVE_LIBFFTW")
+    target_include_directories(Math::FFTW3 INTERFACE "${MKL_FFTW3}")
+    target_link_libraries(Math::FFTW3 INTERFACE "${MKL_LIBRARIES}")
   endif(HAVE_MKL_FFTW3)
 else(HAVE_MKL)
   set(MKL_FOUND FALSE)

CMake/FindMKLVML.cmake

@@ -0,0 +1,25 @@
+# Simple file to verify MKL VML
+# Input: MKL_FOUND, MKL_INCLUDE, MKL_LIBRARIES
+# Output: HAVE_MKL_VML
+
+message(STATUS "Setting up Intel MKL Vector Math Library (VML)")
+
+if(NOT MKL_FOUND)
+  message(FATAL_ERROR "MKL was not found. Cannot proceed with Vector Math Library (VML) searching!")
+endif()
+
+include(CheckIncludeFileCXX)
+
+# Check for mkl_vml_functions.h
+set(CMAKE_REQUIRED_INCLUDES "${MKL_INCLUDE}")
+check_include_file_cxx(mkl_vml_functions.h HAVE_MKL_VML)
+unset(CMAKE_REQUIRED_INCLUDES)
+
+if(HAVE_MKL_VML)
+  target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MKL;HAVE_MKL_VML")
+  target_include_directories(Math::scalar_vector_functions INTERFACE "${MKL_INCLUDE}")
+  target_link_libraries(Math::scalar_vector_functions INTERFACE "${MKL_LIBRARIES}")
+  message(STATUS "MKL VML found")
+else()
+  message(FATAL_ERROR "MKL VML not found")
+endif(HAVE_MKL_VML)

CMakeLists.txt

@@ -579,11 +579,20 @@ message(STATUS "LAPACK libraries: ${LAPACK_LIBRARIES}")
 set_target_properties(Math::BLAS_LAPACK PROPERTIES INTERFACE_LINK_LIBRARIES
                                                    "${LAPACK_LINKER_FLAGS};${LAPACK_LIBRARIES}")
 
+#create scalar_vector_functions target
+add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
+#create fftw3 target
+add_library(Math::FFTW3 INTERFACE IMPORTED)
+
 # Detects MKL header files and other components.
 if(LAPACK_LIBRARIES MATCHES "mkl_core")
   include(CMake/FindMKL.cmake)
 endif()
 
+if(MKL_FOUND)
+  set(MATH_VENDOR_GUESS "INTEL_VML")
+endif()
+
 # AFQMC requires MKL sparse for good performance (roughly a factor of 2x)
 if(BUILD_AFQMC AND NOT MKL_FOUND)
   message(
@@ -595,19 +604,29 @@ endif()
 #-------------------------------------------------------------------
 #  set up scalar/vector math libraries
 #-------------------------------------------------------------------
-set(ENABLE_MASS
-    FALSE
-    CACHE BOOL "ENABLE MASS scalar and vector math libraries for Power architecture")
+set(VALID_MATH_VENDORS "GENERIC" "INTEL_VML" "IBM_MASS" "AMD_LIBM")
+if(NOT DEFINED MATH_VENDOR_GUESS)
+  set(MATH_VENDOR_GUESS "GENERIC")
+endif()
+set(QMC_MATH_VENDOR
+    ${MATH_VENDOR_GUESS}
+    CACHE STRING "Vendor optimized libraries for scalar and vector math functions")
+
+# Perform math vendor option check
+if(NOT QMC_MATH_VENDOR IN_LIST VALID_MATH_VENDORS)
+  message(FATAL_ERROR "Invalid vendor math library ${QMC_MATH_VENDOR}, value must be one of ${VALID_MATH_VENDORS}")
+else()
+  message(STATUS "Selected vendor math library ${QMC_MATH_VENDOR}")
+endif()
 
 # This needs to go before HAVE_SINCOS
-if(ENABLE_MASS)
+if(QMC_MATH_VENDOR STREQUAL "INTEL_VML")
+  include(CMake/FindMKLVML.cmake)
+elseif(QMC_MATH_VENDOR STREQUAL "IBM_MASS")
   include(CMake/FindIBMMASS.cmake)
-endif(ENABLE_MASS)
-
-if(NOT MASS_FOUND AND NOT HAVE_MKL_VML)
-  message(STATUS "No usable vector math library detected.")
-  add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
-endif(NOT MASS_FOUND AND NOT HAVE_MKL_VML)
+elseif(QMC_MATH_VENDOR STREQUAL "AMD_LIBM")
+  include(CMake/FindAMDLibM.cmake)
+endif()
 
 # CheckSincos relies on SINCOS_INCLUDE which may be modified based on vendor libraries
 if(NOT SINCOS_INCLUDE)

config/build_olcf_summit_Clang.sh

@@ -23,7 +23,7 @@ if [[ ! -d /ccs/proj/mat151/opt/modules ]] ; then
   exit 1
 fi
 module use /ccs/proj/mat151/opt/modules
-module load llvm/master-latest
+module load llvm/main-20210811
 
 #the XL built fftw is buggy, use the gcc version
 #module load fftw
@@ -39,7 +39,7 @@ for name in offload_cuda_real offload_cuda_real_MP offload_cuda_cplx offload_cud
             cpu_real cpu_real_MP cpu_cplx cpu_cplx_MP
 do
 
-CMAKE_FLAGS="-D CMAKE_BUILD_TYPE=$TYPE -D ENABLE_MASS=1 -D MASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1 -D MPIEXEC_EXECUTABLE=`which jsrun` -D MPIEXEC_NUMPROC_FLAG='-n' -D MPIEXEC_PREFLAGS='-c;16;-g;1;-b;packed:16;--smpiargs=off'"
+CMAKE_FLAGS="-D CMAKE_BUILD_TYPE=$TYPE -D QMC_MATH_VENDOR=IBM_MASS -D MASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1 -D MPIEXEC_EXECUTABLE=`which jsrun` -D MPIEXEC_NUMPROC_FLAG='-n' -D MPIEXEC_PREFLAGS='-c;16;-g;1;-b;packed:16;--smpiargs=off'"
 
 if [[ $name == *"cplx"* ]]; then
   CMAKE_FLAGS="$CMAKE_FLAGS -D QMC_COMPLEX=1"