Adds perf to the CMake build system

.github/workflows/compile_test.yaml

@@ -12,15 +12,17 @@ jobs:
       fail-fast: false
       matrix:
         flux: [ 0.52.0, 0.49.0]
-        mode: ["FLUX_RPC", "UCX"]
+        dtl_mode: ["FLUX_RPC", "UCX"]
+        profiler_mode: ["CALIPER", "NONE"]
         test_mode: ["c", "cpp"] #, "python"]
     runs-on: ubuntu-20.04 # Docker-based jobs must run on Ubuntu
     env:
       FLUX_VERSION: ${{ matrix.flux }}
       SPACK_DIR: "/home/runner/work/spack"
       DYAD_INSTALL_PREFIX: "/home/runner/work/dyad/install"
       DYAD_KVS_NAMESPACE: "test"
-      DYAD_DTL_MODE: ${{ matrix.mode }}
+      DYAD_DTL_MODE: ${{ matrix.dtl_mode }}
+      DYAD_PROFILER_MODE: ${{ matrix.profiler_mode }}
       DYAD_PATH: "/home/runner/work/dyad/temp"
       DYAD_TEST_MODE: ${{ matrix.test_mode }}
     steps:
@@ -191,11 +193,17 @@ jobs:
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack install -j4 [email protected]
           fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            spack install -j4 caliper
+          fi
           mkdir -p ${DYAD_INSTALL_PREFIX}
           spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION}
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack view --verbose symlink ${DYAD_INSTALL_PREFIX} [email protected]
           fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            spack view --verbose symlink ${DYAD_INSTALL_PREFIX} caliper 
+          fi
       - name: Compile DYAD
         run: |
           echo "Activating spack"
@@ -207,9 +215,14 @@ jobs:
           mkdir build
           cd build
           export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${DYAD_INSTALL_PREFIX}/lib/pkgconfig
-          CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC -DDYAD_PROFILER=NONE"
+          CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC"
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
-            CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX -DDYAD_PROFILER=NONE"
+            CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX"
+          fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=CALIPER"
+          else
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=NONE"
           fi
           cmake -DCMAKE_INSTALL_PREFIX=${DYAD_INSTALL_PREFIX} ${CONFIGURE_FLAGS} -DENABLE_DYAD_DEBUG=ON ..
           make install -j

CMakeLists.txt

@@ -158,6 +158,21 @@ if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT")
         message(FATAL_ERROR "-- [${PROJECT_NAME}] ucx is needed for ${PROJECT_NAME} build")
     endif ()
 endif()
+if(DYAD_PROFILER STREQUAL "CALIPER")
+    find_package(caliper REQUIRED)
+    if (${caliper_FOUND})
+        # For some reason, Caliper sets things a little oddly. The main things to be aware of are:
+        #  * caliper_INCLUDE_DIR (singular) is used instead of caliper_INCLUDE_DIRS
+        #  * caliper_LIB_DIR is used instead of caliper_LIBRARIES
+        #  * There is no variable from caliper-config.cmake that points to the actual library files.
+        #    Caliper expects us to use CMake targets for actual linking
+        message(STATUS "[${PROJECT_NAME}] found caliper at ${caliper_INCLUDE_DIR}")
+        include_directories(${caliper_INCLUDE_DIR})
+        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${caliper_LID_DIR})
+    else ()
+        message(FATAL_ERROR "-- [${PROJECT_NAME}] caliper is needed for ${PROJECT_NAME} build")
+    endif ()
+endif()
 if(DYAD_DATA_PLANE STREQUAL "UCX")
     find_package(ucx 1.6 REQUIRED)
     if (${ucx_FOUND})

src/dyad/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(utils)
+add_subdirectory(perf)
 add_subdirectory(dtl)
 add_subdirectory(core)
 add_subdirectory(modules)

src/dyad/core/CMakeLists.txt

@@ -4,7 +4,7 @@ set(DYAD_CORE_PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/dyad_core.h)
 
 add_library(dyad_core SHARED ${DYAD_CORE_SRC})
 target_link_libraries(dyad_core PUBLIC Jansson::Jansson flux::core)
-target_link_libraries(dyad_core PRIVATE utils murmur3 dyad_dtl)
+target_link_libraries(dyad_core PRIVATE utils murmur3 ${PROJECT_NAME}_dtl)
 target_compile_definitions(dyad_core PUBLIC BUILDING_DYAD=1)
 set_target_properties(dyad_core PROPERTIES PUBLIC_HEADER ${DYAD_CORE_PUBLIC_HEADER})
 

src/dyad/dtl/CMakeLists.txt

@@ -1,7 +1,6 @@
 # DTL Interface
 set(DTL_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.c)
-set(DTL_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.h
-                        ${CMAKE_CURRENT_SOURCE_DIR}/dyad_rc.h)
+set(DTL_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.h)
 set(DTL_PUBLIC_HEADERS )
 
 # Flux implementation for DTL
@@ -24,7 +23,7 @@ set(DTL_PRIVATE_HEADERS ${DTL_PRIVATE_HEADERS} ${FLUX_PRIVATE_HEADERS})
 set(DTL_PUBLIC_HEADERS ${DTL_PUBLIC_HEADERS} ${FLUX_PUBLIC_HEADERS})
 
 add_library(${PROJECT_NAME}_dtl SHARED ${DTL_SRC})
-target_link_libraries(${PROJECT_NAME}_dtl PUBLIC utils Jansson::Jansson flux::core)
+target_link_libraries(${PROJECT_NAME}_dtl PUBLIC utils Jansson::Jansson flux::core ${PROJECT_NAME}_perf)
 
 if(DYAD_DATA_PLANE STREQUAL "UCX")
     target_link_libraries(${PROJECT_NAME}_dtl PUBLIC ucx::ucp ucx::ucs)

src/dyad/modules/CMakeLists.txt

@@ -5,7 +5,7 @@ set(DYAD_MODULE_PUBLIC_HEADERS )
 add_library(dyad SHARED ${DYAD_MODULE_SRC})
 set_target_properties(dyad PROPERTIES PREFIX "")
 target_link_libraries(dyad PUBLIC Jansson::Jansson flux::core)
-target_link_libraries(dyad PRIVATE dyad_dtl)
+target_link_libraries(dyad PRIVATE ${PROJECT_NAME}_dtl)
 target_compile_definitions(dyad PUBLIC BUILDING_DYAD=1)
 
 if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT")

src/dyad/perf/CMakeLists.txt

@@ -0,0 +1,41 @@
+set(DYAD_PERF_SRC )
+set(DYAD_PERF_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_perf.h)
+set(DYAD_PERF_PUBLIC_HEADERS )
+
+set(CALIPER_PERF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/caliper_perf.cpp)
+set(CALIPER_PERF_PRIVATE_HEADERS )
+set(CALIPER_PERF_PUBLIC_HEADERS )
+
+set(DEFAULT_PERF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_perf.c)
+set(DEFAULT_PERF_PRIVATE_HEADERS )
+set(DEFAULT_PERF_PUBLIC_HEADERS )
+
+if (DYAD_PROFILER STREQUAL "CALIPER")
+    set(DYAD_PERF_SRC ${DYAD_PERF_SRC} ${CALIPER_PERF_SRC})
+    set(DYAD_PERF_PRIVATE_HEADERS ${DYAD_PERF_PRIVATE_HEADERS} ${CALIPER_PERF_PRIVATE_HEADERS})
+    set(DYAD_PERF_PUBLIC_HEADERS ${DYAD_PERF_PUBLIC_HEADERS} ${CALIPER_PERF_PUBLIC_HEADERS})
+else ()
+    set(DYAD_PERF_SRC ${DYAD_PERF_SRC} ${DEFAULT_PERF_SRC})
+    set(DYAD_PERF_PRIVATE_HEADERS ${DYAD_PERF_PRIVATE_HEADERS} ${DEFAULT_PERF_PRIVATE_HEADERS})
+    set(DYAD_PERF_PUBLIC_HEADERS ${DYAD_PERF_PUBLIC_HEADERS} ${DEFAULT_PERF_PUBLIC_HEADERS})
+endif ()
+
+add_library(${PROJECT_NAME}_perf SHARED ${DYAD_PERF_SRC})
+target_link_libraries(${PROJECT_NAME}_perf PUBLIC flux::core flux::optparse)
+
+if (DYAD_PROFILER STREQUAL "CALIPER")
+    target_link_libraries(${PROJECT_NAME}_perf PUBLIC caliper)
+    target_compile_definitions(${PROJECT_NAME}_perf PUBLIC WITH_CALIPER=1)
+endif ()
+
+install(
+    TARGETS ${PROJECT_NAME}_perf
+    EXPORT ${DYAD_EXPORTED_TARGETS}
+    LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR}
+    ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR}
+    RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR}
+)
+
+if(NOT ${DYAD_PERF_PUBLIC_HEADERS} STREQUAL "")
+    dyad_install_headers("${DYAD_PERF_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR})
+endif()