ecmwf-ifs · awnawab · Jan 24, 2025 · Jan 24, 2025 · Jan 28, 2025 · Jan 27, 2025
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,4 @@
 field_module.F90
 *.swp
 build
+**__pycache__
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -33,24 +33,8 @@ ecbuild_find_package(OpenMP COMPONENTS Fortran REQUIRED)
 ## find fypp
 field_api_find_fypp()
 
-## find OpenACC
-if( ${CMAKE_VERSION} VERSION_LESS "3.25" )
-  if ( FIELD_API_ENABLE_ACC OR (NOT DEFINED FIELD_API_ENABLE_ACC AND (ENABLE_ACC OR NOT DEFINED ENABLE_ACC)) )
-    # See https://gitlab.kitware.com/cmake/cmake/-/issues/23691, fixed in CMake 3.25
-    # (TL;DR: FindOpenACC sets OpenACC_<LANG>_FOUND correctly but does not set
-    #  OpenACC_FOUND unless all three C, CXX, and Fortran have been found - even if
-    #  only one language has been requested via COMPONENTS)
-    find_package( OpenACC COMPONENTS Fortran )
-    if( OpenACC_Fortran_FOUND )
-      set( OpenACC_FOUND ON )
-    endif()
-  endif()
-endif()
-ecbuild_add_option( FEATURE ACC
-                    DEFAULT ON
-                    DESCRIPTION "Support for using GPUs with OpenACC"
-                    REQUIRED_PACKAGES "OpenACC COMPONENTS Fortran" 
-                    CONDITION CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC")
+## determine GPU offload model
+field_api_get_offload_model()
 
 ## set general compiler flags
 field_api_compile_options()
@@ -75,13 +59,6 @@ endif()
 ## find fiat
 field_api_find_fiat_modules()
 
-## check for CUDA
-include(CheckLanguage)
-check_language(CUDA)
-ecbuild_add_option( FEATURE CUDA
-    DESCRIPTION "CUDA" DEFAULT ON
-    CONDITION CMAKE_CUDA_COMPILER AND HAVE_ACC )
-
 ## buddy allocator option
 ecbuild_add_option( FEATURE BUDDY_MALLOC
     DESCRIPTION "Use buddy allocator for shadow host allocation"
@@ -100,7 +77,7 @@ if(HAVE_BUDDY_MALLOC)
   list( APPEND fypp_defines "-DUSE_BUDDY_MALLOC")
 endif()
 if(HAVE_CUDA)
-  list( APPEND fypp_defines "-DCUDA")
+  list( APPEND fypp_defines "-DWITH_HIC")
 endif()
 if(fiat_FOUND)
   list( APPEND fypp_defines "-DWITH_FIAT")

diff --git a/cmake/field_api_add_object_library.cmake b/cmake/field_api_add_object_library.cmake
@@ -54,6 +54,7 @@ macro(field_api_add_object_library)
            ${_PAR_DEFINITIONS}
            $<$<NOT:${fiat_FOUND}>:${FIELD_API_DEFINITIONS}>
            $<${fiat_FOUND}:WITH_FIAT>
+           ${FIELD_API_OFFLOAD_DEFINITIONS}
         PRIVATE_LIBS
            ${_PAR_LIBRARIES}
            $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>

diff --git a/cmake/field_api_expand_fypp.cmake b/cmake/field_api_expand_fypp.cmake
@@ -41,7 +41,9 @@ macro( field_api_expand_fypp )
     foreach (SRC ${_PAR_INPUT_SRCS} )
 
       add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${SRC}.F90
-    	  COMMAND ${FYPP} -m os ${fypp_defines} -M ${_PAR_PYTHON_MODULE_DIR} -m fieldType ${_PAR_SOURCE_DIR}/${SRC}.fypp > ${CMAKE_CURRENT_BINARY_DIR}/${SRC}.F90
+    	  COMMAND ${FYPP} -m os ${fypp_defines} -M ${_PAR_PYTHON_MODULE_DIR} -m fieldType
+                        -DOFFLOAD_MODEL="${FIELD_API_OFFLOAD_MODEL}" -M ${_PAR_PYTHON_MODULE_DIR} -m offload_macros
+                        ${_PAR_SOURCE_DIR}/${SRC}.fypp > ${CMAKE_CURRENT_BINARY_DIR}/${SRC}.F90
         DEPENDS ${_PAR_SOURCE_DIR}/${SRC}.fypp
         VERBATIM)
 

diff --git a/cmake/field_api_expand_fypp_ranksuff.cmake b/cmake/field_api_expand_fypp_ranksuff.cmake
@@ -51,6 +51,7 @@ macro( field_api_expand_fypp_ranksuff )
 
              add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/field_${RANK}${suff}${FUNC}_module.F90
        	      COMMAND ${FYPP} -DRANK=${RANK} -DSUFF='${SUFF}' ${fypp_defines} -m os -M ${_PAR_PYTHON_MODULE_DIR} -m fieldType 
+               -DOFFLOAD_MODEL="${FIELD_API_OFFLOAD_MODEL}" -M ${_PAR_PYTHON_MODULE_DIR} -m offload_macros
                ${_PAR_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp > ${CMAKE_CURRENT_BINARY_DIR}/field_${RANK}${suff}${FUNC}_module.F90
                DEPENDS ${_PAR_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp
                VERBATIM)
@@ -62,6 +63,7 @@ macro( field_api_expand_fypp_ranksuff )
 
              add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/field_${RANK}${suff}${FUNC}_module.F90
        	      COMMAND ${FYPP} -DRANK=${RANK} -DSUFF='${SUFF}' ${fypp_defines} -m os -M ${_PAR_PYTHON_MODULE_DIR} -m fieldType 
+               -DOFFLOAD_MODEL="${FIELD_API_OFFLOAD_MODEL}" -M ${_PAR_PYTHON_MODULE_DIR} -m offload_macros
                ${_PAR_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp > ${CMAKE_CURRENT_BINARY_DIR}/field_${RANK}${suff}${FUNC}_module.F90
                DEPENDS ${_PAR_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp
                VERBATIM)

diff --git a/cmake/field_api_find_fypp.cmake b/cmake/field_api_find_fypp.cmake
@@ -26,16 +26,15 @@ macro( field_api_find_fypp )
 
    if( fckit_FOUND AND fckit_HAVE_FCKIT_VENV )
      set( FYPP ${FCKIT_VENV_EXE} -m fypp )
-   elseif( fckit_FOUND )
-     # This is only needed for building in environments with python3 older than 3.8
-     list( APPEND _fckit_fypp_path "${FYPP}" )
-     list( LENGTH _fckit_fypp_path _list_length )
-     MATH( EXPR _last_entry "${_list_length} - 1" )
-     list( GET _fckit_fypp_path ${_last_entry} FYPP )
-   elseif( FYPP MATCHES FYPP-NOTFOUND )
-     include(cmake/field_api_fetchcontent_fypp.cmake)
-     set(FYPP ${fypp_SOURCE_DIR}/bin/fypp)
-     ecbuild_info("fypp downloaded to: ${FYPP}")
+   elseif( FYPP MATCHES FYPP-NOTFOUND OR (fckit_FOUND AND (NOT fckit_HAVE_FCKIT_VENV OR NOT DEFINED fckit_HAVE_FCKIT_VENV)) )
+     # Discover only system install Python 3
+     set( Python3_FIND_VIRTUALENV STANDARD )
+     find_package( Python3 COMPONENTS Interpreter )
+
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m ensurepip --upgrade OUTPUT_QUIET )
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip --disable-pip-version-check install fypp OUTPUT_QUIET )
-     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip --disable-pip-version-check install fypp OUTPUT_QUIET )
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m ensurepip --upgrade OUTPUT_QUIET )
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip --disable-pip-version-check install fypp OUTPUT_QUIET )
-     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip --disable-pip-version-check install fypp OUTPUT_QUIET )
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m ensurepip --upgrade OUTPUT_QUIET )
+     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip --disable-pip-version-check install fypp OUTPUT_QUIET )
+     ecbuild_info("field_api installed fypp as a pip package")
+     set( FYPP ${Python3_EXECUTABLE} -m fypp )
    endif()
 
 endmacro()
diff --git a/cmake/field_api_get_offload_model.cmake b/cmake/field_api_get_offload_model.cmake
@@ -0,0 +1,62 @@
+# (C) Copyright 2022- ECMWF.
+# (C) Copyright 2022- Meteo-France.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+##############################################################################
+#.rst:
+#
+# field_api_get_offload_model
+# ===========================
+# 
+# Determine the GPU offload model to be used. ::
+#
+#        field_api_get_offload_model()
+#
+##############################################################################
+
+macro( field_api_get_offload_model )
+
+   ## find OpenACC
+   if( ${CMAKE_VERSION} VERSION_LESS "3.25" )
+     if ( FIELD_API_ENABLE_ACC OR (NOT DEFINED FIELD_API_ENABLE_ACC AND (ENABLE_ACC OR NOT DEFINED ENABLE_ACC)) )
+       # See https://gitlab.kitware.com/cmake/cmake/-/issues/23691, fixed in CMake 3.25
+       # (TL;DR: FindOpenACC sets OpenACC_<LANG>_FOUND correctly but does not set
+       #  OpenACC_FOUND unless all three C, CXX, and Fortran have been found - even if
+       #  only one language has been requested via COMPONENTS)
+       find_package( OpenACC COMPONENTS Fortran )
+       if( OpenACC_Fortran_FOUND )
+         set( OpenACC_FOUND ON )
+       endif()
+     endif()
+   endif()
+   ecbuild_add_option( FEATURE ACC
+                       DEFAULT ON
+                       DESCRIPTION "Support for using GPUs with OpenACC"
+                       REQUIRED_PACKAGES "OpenACC COMPONENTS Fortran" 
+                       CONDITION CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC")
+
+   ## check for CUDA
+   include(CheckLanguage)
+   check_language(CUDA)
+   ecbuild_add_option( FEATURE CUDA
+       DESCRIPTION "CUDA" DEFAULT ON
+       CONDITION CMAKE_CUDA_COMPILER AND HAVE_ACC )
+
+   set(FIELD_API_OFFLOAD_MODEL "HostOnly")
+   if( HAVE_CUDA )
+      set(FIELD_API_OFFLOAD_MODEL "NVHPCOpenACCCUDA")
+   elseif( HAVE_ACC )
+      set(FIELD_API_OFFLOAD_MODEL "NVHPCOpenACC")
+   endif()
+
+   unset(FIELD_API_OFFLOAD_DEFINITIONS)
+   if( HAVE_ACC )
+      list(APPEND FIELD_API_OFFLOAD_DEFINITIONS WITH_GPU_OFFLOAD)
+   endif()
+
+endmacro()
diff --git a/cmake/field_api_macros.cmake b/cmake/field_api_macros.cmake
@@ -14,3 +14,4 @@ include( field_api_expand_fypp )
 include( field_api_expand_fypp_ranksuff )
 include( field_api_add_object_library )
 include( field_api_target_add_module_dirs )
+include( field_api_get_offload_model )
diff --git a/python_utils/offload_backends/__init__.py b/python_utils/offload_backends/__init__.py
@@ -0,0 +1,11 @@
+# (C) Copyright 2022- ECMWF.
+# (C) Copyright 2022- Meteo-France.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from offload_backends.nvhpc import *
+from offload_backends.host_only import *
diff --git a/cmake/field_api_fetchcontent_fypp.cmake → python_utils/offload_backends/host_only.py b/cmake/field_api_fetchcontent_fypp.cmake → python_utils/offload_backends/host_only.py
@@ -7,14 +7,12 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-# Download fypp preprocessor if not found.
 
-include(FetchContent)
+__all__ = ['HostOnly']
 
-FetchContent_Declare(
-   fypp
-   GIT_REPOSITORY https://github.com/aradi/fypp
-   GIT_TAG 3.1
-)
+class HostOnly():
+    """
+    A dummy class only to be used if GPU offload is disabled.
+    """
 
-FetchContent_MakeAvailable(fypp)
+    pragma = ''
diff --git a/python_utils/offload_backends/nvhpc/__init__.py b/python_utils/offload_backends/nvhpc/__init__.py
@@ -0,0 +1,11 @@
+# (C) Copyright 2022- ECMWF.
+# (C) Copyright 2022- Meteo-France.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from offload_backends.nvhpc.openacc import *
+from offload_backends.nvhpc.openacc_cuda import *
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,3 +5,4 @@ @@
     field_module.F90
     *.swp
     build
+    **__pycache__