Merge pull request #153 from amametjanov/omega/fix-sycl-runs

Fix CTests in SYCL runs
E3SM-Project · Jan 29, 2025 · cc4eb05 · cc4eb05
2 parents c6f79c6 + c2fb28b
commit cc4eb05
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 22 deletions.
diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake
@@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
 endif()
-string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ")
+string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ")
 string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake
@@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
 endif()
-string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ")
+string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ")
 string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt
@@ -44,7 +44,7 @@ if (NOT DEFINED PROJECT_NAME)
 
   set(CMAKE_CXX_STANDARD 17) # used in E3SM
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
-  set(LINKER_LANGUAGE C)     # needed to support Cray C compiler wrapper
+  set(LINKER_LANGUAGE CXX)   # needed to support Cray C compiler wrapper
 
   # update variables for standalone build
   setup_standalone_build()

diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake
@@ -30,6 +30,10 @@ macro(common)
   option(OMEGA_LOG_FLUSH "Turn on unbuffered logging (default OFF)." OFF)
   option(OMEGA_TEST_CDASH "Turn on CDash support (default ON)." ON)
 
+  if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG")
+    set(OMEGA_DEBUG ON)
+  endif()
+
   if(NOT DEFINED OMEGA_CXX_FLAGS)
     set(OMEGA_CXX_FLAGS "")
   endif()
@@ -123,7 +127,7 @@ macro(read_cime_config)
                 break()
 
             elseif("${arg}" STREQUAL "-n" OR "${arg}" STREQUAL "-N" OR
-                   "${arg}" STREQUAL "-c")
+                   "${arg}" STREQUAL "-c" OR "${arg}" STREQUAL "-np")
                 set(SKIP_ARG TRUE)
 
             else()
@@ -222,6 +226,9 @@ macro(init_standalone_build)
     elseif(USE_HIP)
       set(OMEGA_ARCH "HIP")
 
+    elseif(USE_SYCL)
+      set(OMEGA_ARCH "SYCL")
+
     else()
 
       execute_process(
@@ -300,7 +307,11 @@ macro(init_standalone_build)
   set(_CtestScript ${OMEGA_BUILD_DIR}/omega_ctest.sh)
   file(WRITE ${_CtestScript}  "#!/usr/bin/env bash\n\n")
   file(APPEND ${_CtestScript} "source ./omega_env.sh\n\n")
-  file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n")
+  if(OMEGA_DEBUG)
+    file(APPEND ${_CtestScript} "ctest --output-on-failure --verbose $* # --rerun-failed\n\n")
+  else()
+    file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n")
+  endif()
 
   # create a profile script
   set(_ProfileScript ${OMEGA_BUILD_DIR}/omega_profile.sh)
@@ -428,11 +439,18 @@ macro(init_standalone_build)
     file(APPEND ${_ProfileScript} "    -o \$OUTFILE ./src/omega.exe 1000")
 
   elseif("${OMEGA_ARCH}" STREQUAL "SYCL")
-    set(CMAKE_CXX_COMPILER ${OMEGA_SYCL_COMPILER})
+    set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
 
+    # add flags from upstream-E3SM
+    if(SYCL_FLAGS)
+      set(OMEGA_SYCL_FLAGS "${OMEGA_SYCL_FLAGS} ${SYCL_FLAGS}")
+    endif()
     if(OMEGA_SYCL_FLAGS)
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_SYCL_FLAGS}")
     endif()
+    if(OMEGA_SYCL_EXE_LINKER_FLAGS)
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMEGA_SYCL_EXE_LINKER_FLAGS}")
+    endif()
 
   else()
     set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
@@ -464,7 +482,7 @@ macro(init_standalone_build)
 
   message(STATUS "CMAKE_CXX_COMPILER     = ${CMAKE_CXX_COMPILER}")
   message(STATUS "CMAKE_CXX_FLAGS        = ${CMAKE_CXX_FLAGS}")
-#  message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
+  message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
 
 endmacro()
 
@@ -518,10 +536,6 @@ macro(update_variables)
 
   add_definitions(-DOMEGA_BUILD_MODE=${OMEGA_BUILD_MODE})
 
-  if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG")
-    set(OMEGA_DEBUG ON)
-  endif()
-
   if(NOT DEFINED OMEGA_LOG_LEVEL)
     set(OMEGA_LOG_LEVEL "INFO")
   endif()

diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt
@@ -54,7 +54,7 @@ target_link_libraries(
 if(GKlib_FOUND)
     target_link_libraries(
         OmegaLibFlags
-        PUBLIC
+        INTERFACE
         gklib
     )
 endif()
@@ -107,6 +107,6 @@ if(OMEGA_BUILD_EXECUTABLE)
     OmegaLibFlags
   )
 
-  set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE C)
+  set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX)
 
 endif()
diff --git a/components/omega/src/base/DataTypes.h b/components/omega/src/base/DataTypes.h
@@ -43,6 +43,8 @@ KOKKOS_INLINE_FUNCTION constexpr Real operator""_Real(long double x) {
 using MemSpace = Kokkos::CudaSpace;
 #elif OMEGA_ENABLE_HIP
 using MemSpace = Kokkos::Experimental::HIPSpace;
+#elif OMEGA_ENABLE_SYCL
+using MemSpace = Kokkos::Experimental::SYCLDeviceUSMSpace;
 #elif OMEGA_ENABLE_OPENMP
 using MemSpace = Kokkos::HostSpace;
 #elif OMEGA_ENABLE_SERIAL

diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt
@@ -45,14 +45,22 @@ function(add_omega_test test_name exe_name source_files mpi_args)
     )
   endif()
 
-  set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE C)
+  set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE CXX)
 
   # Add the test command
   if (mpi_args)
-    add_test(
-      NAME ${test_name}
-      COMMAND ../omega_env.sh ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name}
-    )
+
+    if("${OMEGA_ARCH}" STREQUAL "SYCL")
+      add_test(
+        NAME ${test_name}
+        COMMAND ${OMEGA_MPI_EXEC} ${mpi_args} ${OMEGA_MPI_ARGS} ./${exe_name}
+      )
+    else()
+      add_test(
+        NAME ${test_name}
+        COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name}
+      )
+    endif()
 
   else()
     add_test(
@@ -282,7 +290,7 @@ add_omega_test(
     TEND_PLANE_TEST
     testTendencyTermsPlane.exe
     ocn/TendencyTermsTest.cpp
-    "-n 8;--cpu-bind=cores"
+    "-n 8;"
 )
 target_compile_definitions(
   testTendencyTermsPlane.exe
@@ -294,15 +302,15 @@ add_omega_test(
     TEND_PLANE_SINGLE_PRECISION_TEST
     testTendencyTermsPlaneSinglePrecision.exe
     ocn/TendencyTermsTest.cpp
-    "-n 8;--cpu-bind=cores"
+    "-n 8;"
     single_precision
 )
 
 add_omega_test(
   TEND_SPHERE_TEST
     testTendencyTermsSphere.exe
     ocn/TendencyTermsTest.cpp
-    "-n 8;--cpu-bind=cores"
+    "-n 8;"
 )
 target_compile_definitions(
   testTendencyTermsSphere.exe
@@ -351,7 +359,7 @@ add_omega_test(
     REDUCTIONS_TEST
     testReductions.exe
     base/ReductionsTest.cpp
-    "-n;2;--cpu-bind=cores"
+    "-n;2;"
 )
 
 ###################