diff --git a/.github/workflows/compile_test.yaml b/.github/workflows/compile_test.yaml index e26804ad..0602cc91 100644 --- a/.github/workflows/compile_test.yaml +++ b/.github/workflows/compile_test.yaml @@ -12,7 +12,8 @@ jobs: fail-fast: false matrix: flux: [ 0.52.0, 0.49.0] - mode: ["FLUX_RPC", "UCX"] + dtl_mode: ["FLUX_RPC", "UCX"] + profiler_mode: ["CALIPER", "NONE"] test_mode: ["c", "cpp"] #, "python"] runs-on: ubuntu-20.04 # Docker-based jobs must run on Ubuntu env: @@ -20,7 +21,8 @@ jobs: SPACK_DIR: "/home/runner/work/spack" DYAD_INSTALL_PREFIX: "/home/runner/work/dyad/install" DYAD_KVS_NAMESPACE: "test" - DYAD_DTL_MODE: ${{ matrix.mode }} + DYAD_DTL_MODE: ${{ matrix.dtl_mode }} + DYAD_PROFILER_MODE: ${{ matrix.profiler_mode }} DYAD_PATH: "/home/runner/work/dyad/temp" DYAD_TEST_MODE: ${{ matrix.test_mode }} steps: @@ -191,11 +193,17 @@ jobs: if [[ $DYAD_DTL_MODE == 'UCX' ]]; then spack install -j4 ucx@1.13.1 fi + if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then + spack install -j4 caliper + fi mkdir -p ${DYAD_INSTALL_PREFIX} spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION} if [[ $DYAD_DTL_MODE == 'UCX' ]]; then spack view --verbose symlink ${DYAD_INSTALL_PREFIX} ucx@1.13.1 fi + if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then + spack view --verbose symlink ${DYAD_INSTALL_PREFIX} caliper + fi - name: Compile DYAD run: | echo "Activating spack" @@ -207,9 +215,14 @@ jobs: mkdir build cd build export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${DYAD_INSTALL_PREFIX}/lib/pkgconfig - CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC -DDYAD_PROFILER=NONE" + CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC" if [[ $DYAD_DTL_MODE == 'UCX' ]]; then - CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX -DDYAD_PROFILER=NONE" + CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX" + fi + if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then + CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=CALIPER" + else + CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=NONE" fi cmake -DCMAKE_INSTALL_PREFIX=${DYAD_INSTALL_PREFIX} ${CONFIGURE_FLAGS} -DENABLE_DYAD_DEBUG=ON .. make install -j diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e9f4bce..0660fa6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,6 +158,21 @@ if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT") message(FATAL_ERROR "-- [${PROJECT_NAME}] ucx is needed for ${PROJECT_NAME} build") endif () endif() +if(DYAD_PROFILER STREQUAL "CALIPER") + find_package(caliper REQUIRED) + if (${caliper_FOUND}) + # For some reason, Caliper sets things a little oddly. The main things to be aware of are: + # * caliper_INCLUDE_DIR (singular) is used instead of caliper_INCLUDE_DIRS + # * caliper_LIB_DIR is used instead of caliper_LIBRARIES + # * There is no variable from caliper-config.cmake that points to the actual library files. + # Caliper expects us to use CMake targets for actual linking + message(STATUS "[${PROJECT_NAME}] found caliper at ${caliper_INCLUDE_DIR}") + include_directories(${caliper_INCLUDE_DIR}) + set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${caliper_LID_DIR}) + else () + message(FATAL_ERROR "-- [${PROJECT_NAME}] caliper is needed for ${PROJECT_NAME} build") + endif () +endif() if(DYAD_DATA_PLANE STREQUAL "UCX") find_package(ucx 1.6 REQUIRED) if (${ucx_FOUND}) diff --git a/src/dyad/CMakeLists.txt b/src/dyad/CMakeLists.txt index 7a5f5655..e1ea3a36 100644 --- a/src/dyad/CMakeLists.txt +++ b/src/dyad/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(utils) +add_subdirectory(perf) add_subdirectory(dtl) add_subdirectory(core) add_subdirectory(modules) diff --git a/src/dyad/core/CMakeLists.txt b/src/dyad/core/CMakeLists.txt index 6750cf7e..21b59975 100644 --- a/src/dyad/core/CMakeLists.txt +++ b/src/dyad/core/CMakeLists.txt @@ -4,7 +4,7 @@ set(DYAD_CORE_PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/dyad_core.h) add_library(dyad_core SHARED ${DYAD_CORE_SRC}) target_link_libraries(dyad_core PUBLIC Jansson::Jansson flux::core) -target_link_libraries(dyad_core PRIVATE utils murmur3 dyad_dtl) +target_link_libraries(dyad_core PRIVATE utils murmur3 ${PROJECT_NAME}_dtl) target_compile_definitions(dyad_core PUBLIC BUILDING_DYAD=1) set_target_properties(dyad_core PROPERTIES PUBLIC_HEADER ${DYAD_CORE_PUBLIC_HEADER}) diff --git a/src/dyad/dtl/CMakeLists.txt b/src/dyad/dtl/CMakeLists.txt index 30f1ecb5..63a1e34b 100644 --- a/src/dyad/dtl/CMakeLists.txt +++ b/src/dyad/dtl/CMakeLists.txt @@ -1,7 +1,6 @@ # DTL Interface set(DTL_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.c) -set(DTL_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.h - ${CMAKE_CURRENT_SOURCE_DIR}/dyad_rc.h) +set(DTL_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_impl.h) set(DTL_PUBLIC_HEADERS ) # Flux implementation for DTL @@ -24,7 +23,7 @@ set(DTL_PRIVATE_HEADERS ${DTL_PRIVATE_HEADERS} ${FLUX_PRIVATE_HEADERS}) set(DTL_PUBLIC_HEADERS ${DTL_PUBLIC_HEADERS} ${FLUX_PUBLIC_HEADERS}) add_library(${PROJECT_NAME}_dtl SHARED ${DTL_SRC}) -target_link_libraries(${PROJECT_NAME}_dtl PUBLIC utils Jansson::Jansson flux::core) +target_link_libraries(${PROJECT_NAME}_dtl PUBLIC utils Jansson::Jansson flux::core ${PROJECT_NAME}_perf) if(DYAD_DATA_PLANE STREQUAL "UCX") target_link_libraries(${PROJECT_NAME}_dtl PUBLIC ucx::ucp ucx::ucs) diff --git a/src/dyad/modules/CMakeLists.txt b/src/dyad/modules/CMakeLists.txt index a92b2bd4..8b769b60 100644 --- a/src/dyad/modules/CMakeLists.txt +++ b/src/dyad/modules/CMakeLists.txt @@ -5,7 +5,7 @@ set(DYAD_MODULE_PUBLIC_HEADERS ) add_library(dyad SHARED ${DYAD_MODULE_SRC}) set_target_properties(dyad PROPERTIES PREFIX "") target_link_libraries(dyad PUBLIC Jansson::Jansson flux::core) -target_link_libraries(dyad PRIVATE dyad_dtl) +target_link_libraries(dyad PRIVATE ${PROJECT_NAME}_dtl) target_compile_definitions(dyad PUBLIC BUILDING_DYAD=1) if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT") diff --git a/src/dyad/perf/CMakeLists.txt b/src/dyad/perf/CMakeLists.txt new file mode 100644 index 00000000..6617a447 --- /dev/null +++ b/src/dyad/perf/CMakeLists.txt @@ -0,0 +1,41 @@ +set(DYAD_PERF_SRC ) +set(DYAD_PERF_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_perf.h) +set(DYAD_PERF_PUBLIC_HEADERS ) + +set(CALIPER_PERF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/caliper_perf.cpp) +set(CALIPER_PERF_PRIVATE_HEADERS ) +set(CALIPER_PERF_PUBLIC_HEADERS ) + +set(DEFAULT_PERF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_perf.c) +set(DEFAULT_PERF_PRIVATE_HEADERS ) +set(DEFAULT_PERF_PUBLIC_HEADERS ) + +if (DYAD_PROFILER STREQUAL "CALIPER") + set(DYAD_PERF_SRC ${DYAD_PERF_SRC} ${CALIPER_PERF_SRC}) + set(DYAD_PERF_PRIVATE_HEADERS ${DYAD_PERF_PRIVATE_HEADERS} ${CALIPER_PERF_PRIVATE_HEADERS}) + set(DYAD_PERF_PUBLIC_HEADERS ${DYAD_PERF_PUBLIC_HEADERS} ${CALIPER_PERF_PUBLIC_HEADERS}) +else () + set(DYAD_PERF_SRC ${DYAD_PERF_SRC} ${DEFAULT_PERF_SRC}) + set(DYAD_PERF_PRIVATE_HEADERS ${DYAD_PERF_PRIVATE_HEADERS} ${DEFAULT_PERF_PRIVATE_HEADERS}) + set(DYAD_PERF_PUBLIC_HEADERS ${DYAD_PERF_PUBLIC_HEADERS} ${DEFAULT_PERF_PUBLIC_HEADERS}) +endif () + +add_library(${PROJECT_NAME}_perf SHARED ${DYAD_PERF_SRC}) +target_link_libraries(${PROJECT_NAME}_perf PUBLIC flux::core flux::optparse) + +if (DYAD_PROFILER STREQUAL "CALIPER") + target_link_libraries(${PROJECT_NAME}_perf PUBLIC caliper) + target_compile_definitions(${PROJECT_NAME}_perf PUBLIC WITH_CALIPER=1) +endif () + +install( + TARGETS ${PROJECT_NAME}_perf + EXPORT ${DYAD_EXPORTED_TARGETS} + LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} +) + +if(NOT ${DYAD_PERF_PUBLIC_HEADERS} STREQUAL "") + dyad_install_headers("${DYAD_PERF_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) +endif() \ No newline at end of file