Adds Caliper annotations and adds flux-optparse support for server ar…

…gument parsing

.github/prod-cons/dyad_producer.sh

@@ -8,8 +8,8 @@ export DYAD_PATH_PRODUCER=${DYAD_PATH}_producer
 mkdir -p ${DYAD_PATH_PRODUCER}
 echo "Loading DYAD module"
 
-echo flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  $DYAD_PATH_PRODUCER $DYAD_DTL_MODE
-flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  $DYAD_PATH_PRODUCER $DYAD_DTL_MODE
+echo flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so --dtl_mode=$DYAD_DTL_MODE $DYAD_PATH_PRODUCER
+flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so --dtl_mode=$DYAD_DTL_MODE $DYAD_PATH_PRODUCER
 
 if [[ "$mode" == "${valid_modes[0]}" ]]; then
     echo ${GITHUB_WORKSPACE}/docs/demos/ecp_feb_2023/c_prod 10 $DYAD_PATH_PRODUCER

.github/workflows/compile_test.yaml

@@ -12,15 +12,17 @@ jobs:
       fail-fast: false
       matrix:
         flux: [ 0.52.0, 0.49.0]
-        mode: ["FLUX_RPC", "UCX"]
+        dtl_mode: ["FLUX_RPC", "UCX"]
+        profiler_mode: ["CALIPER", "NONE"]
         test_mode: ["c", "cpp"] #, "python"]
     runs-on: ubuntu-20.04 # Docker-based jobs must run on Ubuntu
     env:
       FLUX_VERSION: ${{ matrix.flux }}
       SPACK_DIR: "/home/runner/work/spack"
       DYAD_INSTALL_PREFIX: "/home/runner/work/dyad/install"
       DYAD_KVS_NAMESPACE: "test"
-      DYAD_DTL_MODE: ${{ matrix.mode }}
+      DYAD_DTL_MODE: ${{ matrix.dtl_mode }}
+      DYAD_PROFILER_MODE: ${{ matrix.profiler_mode }}
       DYAD_PATH: "/home/runner/work/dyad/temp"
       DYAD_TEST_MODE: ${{ matrix.test_mode }}
     steps:
@@ -191,11 +193,17 @@ jobs:
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack install -j4 [email protected]
           fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            spack install -j4 caliper
+          fi
           mkdir -p ${DYAD_INSTALL_PREFIX}
           spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION}
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack view --verbose symlink ${DYAD_INSTALL_PREFIX} [email protected]
           fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            spack view --verbose symlink ${DYAD_INSTALL_PREFIX} caliper 
+          fi
       - name: Compile DYAD
         run: |
           echo "Activating spack"
@@ -207,9 +215,14 @@ jobs:
           mkdir build
           cd build
           export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${DYAD_INSTALL_PREFIX}/lib/pkgconfig
-          CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC -DDYAD_PROFILER=NONE"
+          CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=FLUX_RPC"
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
-            CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX -DDYAD_PROFILER=NONE"
+            CONFIGURE_FLAGS="-DDYAD_CONTROL_PLANE=FLUX_RPC -DDYAD_DATA_PLANE=UCX"
+          fi
+          if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=CALIPER"
+          else
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} -DDYAD_PROFILER=NONE"
           fi
           cmake -DCMAKE_INSTALL_PREFIX=${DYAD_INSTALL_PREFIX} ${CONFIGURE_FLAGS} -DENABLE_DYAD_DEBUG=ON ..
           make install -j

CMakeLists.txt

@@ -158,6 +158,21 @@ if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT")
         message(FATAL_ERROR "-- [${PROJECT_NAME}] ucx is needed for ${PROJECT_NAME} build")
     endif ()
 endif()
+if(DYAD_PROFILER STREQUAL "CALIPER")
+    find_package(caliper REQUIRED)
+    if (${caliper_FOUND})
+        # For some reason, Caliper sets things a little oddly. The main things to be aware of are:
+        #  * caliper_INCLUDE_DIR (singular) is used instead of caliper_INCLUDE_DIRS
+        #  * caliper_LIB_DIR is used instead of caliper_LIBRARIES
+        #  * There is no variable from caliper-config.cmake that points to the actual library files.
+        #    Caliper expects us to use CMake targets for actual linking
+        message(STATUS "[${PROJECT_NAME}] found caliper at ${caliper_INCLUDE_DIR}")
+        include_directories(${caliper_INCLUDE_DIR})
+        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${caliper_LID_DIR})
+    else ()
+        message(FATAL_ERROR "-- [${PROJECT_NAME}] caliper is needed for ${PROJECT_NAME} build")
+    endif ()
+endif()
 if(DYAD_DATA_PLANE STREQUAL "UCX")
     find_package(ucx 1.6 REQUIRED)
     if (${ucx_FOUND})

configure.ac

@@ -81,6 +81,12 @@ AC_ARG_ENABLE([ucx],
     [enable_ucx=no]
 )
 AM_CONDITIONAL([UCX], [test "x$enable_ucx" = "xyes"])
+AC_ARG_ENABLE([caliper],
+    [AS_HELP_STRING([--enable-caliper],
+                    [enable performance measurement with Caliper])],
+    [enable_caliper=$withval],
+    [enable_caliper=no]
+)
 
 #############################################
 # Define PKG_CHECK_VAR if it does not exist #
@@ -120,9 +126,10 @@ AX_PERFFLOW_ASPECT([PERFFLOW],
     [pkg_check_perfflow_found=yes],
     [pkg_check_perfflow_found=no]
 )
-if test "x$enable_perfflow" = "xyes" && test "x$pkg_check_perfflow_found" = "xno"; then
+if test "x$with_perfflow" = "xyes" && test "x$pkg_check_perfflow_found" = "xno"; then
     AC_MSG_ERROR([requested PerfFlow Aspect support, but cannot find PerfFlow Aspect with pkg-config])
 fi
+DYAD_MOD_RPATH=""
 # Check for UCX v1.6.0 or higher
 PKG_CHECK_MODULES([UCX],
     [ucx >= 1.6.0],
@@ -143,11 +150,52 @@ if test "x$enable_ucx" = "xyes"; then
         [AC_MSG_FAILURE([check_var succeeded, but value is incorrect])]
     )
     AS_IF([test "x$enable_ucx" = "xyes"],
-        [DYAD_MOD_RPATH="-Wl,-rpath,$UCX_LIBDIR"],
-        [DYAD_MOD_RPATH=""]
+        [
+            if test -z "$DYAD_MOD_RPATH"; then
+                DYAD_MOD_RPATH="$UCX_LIBDIR"
+            else
+                DYAD_MOD_RPATH="$DYAD_MOD_RPATH:$UCX_LIBDIR"
+            fi
+        ],
+        []
     )
-    AC_SUBST([DYAD_MOD_RPATH])
 fi
+PKG_CHECK_MODULES([CALIPER],
+    [caliper],
+    [pkg_check_caliper_found=yes],
+    [pkg_check_caliper_found=no]
+)
+if test "x$enable_caliper" = "xyes" && test "x$pkg_check_caliper_found" = "xno"; then
+    AC_MSG_ERROR([requested Caliper support, but cannot find Caliper with pkg-config])
+fi
+if test "x$enable_caliper" = "xyes"; then
+    PKG_CHECK_VAR([CALIPER_LIBDIR],
+        [caliper],
+        [libdir],
+        [],
+        [AC_MSG_FAILURE([Could not find libdir for Caliper])]
+    )
+    AS_IF([test "x$CALIPER_LIBDIR" = "x"],
+          [AC_MSG_FAILURE([check_var succeeded, but value is incorrect])]
+    )
+    AS_IF([test "x$enable_caliper" = "xyes"],
+        [
+            if test -z "$DYAD_MOD_RPATH"; then
+                DYAD_MOD_RPATH="$CALIPER_LIBDIR"
+            else
+                DYAD_MOD_RPATH="$DYAD_MOD_RPATH:$CALIPER_LIBDIR"
+            fi
+        ],
+        []
+    )
+fi
+AM_CONDITIONAL([WITH_CALIPER], [test "x$enable_caliper" = "xyes"])
+
+AS_IF([test -z "$DYAD_MOD_RPATH"],
+    [],
+    [DYAD_MOD_RPATH="-Wl,-rpath,$DYAD_MOD_RPATH"]
+)
+AC_SUBST([DYAD_MOD_RPATH])
 
 ###########################
 # Checks for header files #
@@ -207,6 +255,7 @@ fi
 ########################
 AC_CONFIG_FILES([Makefile
                  src/Makefile
+                 src/perf/Makefile
                  src/utils/Makefile
                  src/utils/base64/Makefile
                  src/utils/libtap/Makefile

src/dyad/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(utils)
+add_subdirectory(perf)
 add_subdirectory(dtl)
 add_subdirectory(core)
 add_subdirectory(modules)

src/dyad/Makefile.am

@@ -1 +1 @@
-SUBDIRS = utils dtl modules core wrapper stream
+SUBDIRS = utils perf dtl modules core wrapper stream

src/dyad/common/dyad_rc.h

@@ -7,6 +7,8 @@
 #define DYAD_DLL_EXPORTED
 #endif
 
+#define DYAD_DLL_HIDDEN __attribute__ ((__visibility__ ("default")))
+
 #if DYAD_PERFFLOW
 #define DYAD_PFA_ANNOTATE __attribute__ ((annotate ("@critical_path()")))
 #else
@@ -42,6 +44,9 @@ enum dyad_core_return_codes {
     DYAD_RC_BAD_B64DECODE = -18,   // Decoding of data w/ base64 failed
     DYAD_RC_BAD_COMM_MODE = -19,   // Invalid comm mode provided to DTL
     DYAD_RC_UNTRACKED = -20,       // Provided path is not tracked by DYAD
+    DYAD_RC_PERF_INIT_FAIL = -21,   // Performance measurement initialization failed
+    DYAD_RC_BAD_CLI_ARG_DEF = -22,  // Trying to define a CLI argument failed
+    DYAD_RC_BAD_CLI_PARSE = -23,    // Trying to parse CLI arguments failed
 };
 
 typedef enum dyad_core_return_codes dyad_rc_t;

src/dyad/core/Makefile.am

@@ -9,12 +9,14 @@ libdyad_core_la_CFLAGS = \
 	$(AM_CFLAGS) \
 	-I$(top_srcdir)/src/utils \
 	-I$(top_srcdir)/src/utils/base64 \
+	-I$(top_srcdir)/src/common \
 	-I$(top_srcdir)/src/dtl \
-    $(JANSSON_CFLAGS) \
+	-I$(top_srcdir)/src/perf \
+	$(JANSSON_CFLAGS) \
 	$(FLUX_CORE_CFLAGS) \
 	-DBUILDING_DYAD=1 \
 	-fvisibility=hidden
-libdyad_core_la_CPPFLAGS = 
+libdyad_core_la_CPPFLAGS =
 if UCX
 libdyad_core_la_LIBADD += $(UCX_LIBS)
 libdyad_core_la_CFLAGS += $(UCX_CFLAGS)
@@ -24,5 +26,12 @@ libdyad_core_la_LIBADD += $(PERFFLOW_LIBS)
 libdyad_core_la_CFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1
 libdyad_core_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS)
 endif
+if WITH_CALIPER
+libdyad_core_la_CFLAGS += \
+    -DWITH_CALIPER=1 \
+	$(CALIPER_CFLAGS)
+libdyad_core_la_LIBADD += \
+    $(CALIPER_LIBS)
+endif
 
 include_HEADERS = dyad_core.h dyad_envs.h