From cdec006da32f2e7bab9e6bc0b705d3fef7404ca4 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 18 Oct 2024 16:06:07 -0400 Subject: [PATCH] Applies results of autoformatting to repo + manually divides includes into groups --- include/dyad/stream/dyad_params.hpp | 2 +- include/dyad/stream/dyad_stream_api.hpp | 169 +++++------ include/dyad/stream/dyad_stream_core.hpp | 10 +- src/dyad/common/dyad_dtl.h | 16 +- src/dyad/common/dyad_logging.h | 228 +++++++-------- src/dyad/common/dyad_rc.h | 43 ++- src/dyad/common/dyad_structures.h | 32 +-- src/dyad/core/dyad_core.c | 231 +++++++-------- src/dyad/core/dyad_core.h | 48 ++-- src/dyad/core/dyad_ctx.c | 22 +- src/dyad/core/dyad_ctx.h | 7 +- src/dyad/dtl/dyad_dtl_api.c | 30 +- src/dyad/dtl/dyad_dtl_api.h | 23 +- src/dyad/dtl/flux_dtl.c | 74 +++-- src/dyad/dtl/ucx_dtl.c | 262 +++++++++--------- src/dyad/dtl/ucx_dtl.h | 11 +- src/dyad/dtl/ucx_ep_cache.cpp | 75 ++--- src/dyad/dtl/ucx_ep_cache.h | 19 +- src/dyad/modules/dyad.c | 46 +-- src/dyad/modules/test_opt_parse.c | 123 ++++---- src/dyad/stream/dyad_stream_core.cpp | 79 +++--- src/dyad/utils/base64/base64.c | 4 +- src/dyad/utils/base64/base64.h | 14 +- .../utils/test_cmp_canonical_path_prefix.c | 10 +- src/dyad/utils/test_murmur3.c | 14 +- src/dyad/utils/utils.c | 178 +++++++----- src/dyad/utils/utils.h | 17 +- src/dyad/wrapper/flux_barrier.c | 14 +- 28 files changed, 927 insertions(+), 874 deletions(-) diff --git a/include/dyad/stream/dyad_params.hpp b/include/dyad/stream/dyad_params.hpp index c322a12c..1a3424f1 100644 --- a/include/dyad/stream/dyad_params.hpp +++ b/include/dyad/stream/dyad_params.hpp @@ -24,7 +24,7 @@ namespace dyad struct dyad_params { bool m_debug; /** Indicate if the storage associated with the managed path is shared - * (i.e. visible to all ranks) */ + * (i.e. visible to all ranks) */ bool m_shared_storage; /// Indicate if reinitialization is required even if already initinialized bool m_reinit; diff --git a/include/dyad/stream/dyad_stream_api.hpp b/include/dyad/stream/dyad_stream_api.hpp index d640cb2d..251d95b0 100644 --- a/include/dyad/stream/dyad_stream_api.hpp +++ b/include/dyad/stream/dyad_stream_api.hpp @@ -19,7 +19,7 @@ #include // fsync -#include // realpath +#include // realpath #include #include #include @@ -28,7 +28,6 @@ #include - namespace dyad { @@ -36,8 +35,7 @@ namespace dyad // Enable if _Path is a filesystem::path or experimental::filesystem::path template ().make_preferred ().filename ())> + typename _Path2 = decltype (std::declval<_Path&> ().make_preferred ().filename ())> using dyad_if_fs_path = std::enable_if_t, _Result>; #endif // c++17 filesystem @@ -63,7 +61,7 @@ void fsync_ofstream (std::basic_ofstream<_CharT, _Traits>& os) if (os.is_open ()) { os.flush (); } -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD } template @@ -87,14 +85,15 @@ void fsync_fstream (std::basic_fstream<_CharT, _Traits>& os) if (os.is_open ()) { os.flush (); } -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD } //---------------------------------------------------------------------- #if defined(DYAD_HAS_STD_FSTREAM_FD) //---------------------------------------------------------------------- template -void lock_exclusive_ofstream (std::basic_ofstream<_CharT, _Traits>& os, const dyad_stream_core& core) +void lock_exclusive_ofstream (std::basic_ofstream<_CharT, _Traits>& os, + const dyad_stream_core& core) { class my_filebuf : public std::basic_filebuf<_CharT> { @@ -219,25 +218,25 @@ void unlock_fstream (std::basic_fstream<_CharT, _Traits>& os, const dyad_stream_ } } -#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_,_core_) lock_exclusive_ofstream(_os_,_core_) -#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_,_core_) lock_exclusive_fstream(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_,_core_) lock_shared_ifstream(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_,_core_) lock_shared_fstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_OFSTREAM(_os_,_core_) unlock_ofstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_IFSTREAM(_os_,_core_) unlock_ifstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_FSTREAM(_os_,_core_) unlock_fstream(_os_,_core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_, _core_) lock_exclusive_ofstream (_os_, _core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_, _core_) lock_exclusive_fstream (_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_, _core_) lock_shared_ifstream (_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_, _core_) lock_shared_fstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_OFSTREAM(_os_, _core_) unlock_ofstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_IFSTREAM(_os_, _core_) unlock_ifstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_FSTREAM(_os_, _core_) unlock_fstream (_os_, _core_) -#else // DYAD_HAS_STD_FSTREAM_FD +#else // DYAD_HAS_STD_FSTREAM_FD -#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_,_core_) -#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_OFSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_IFSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_FSTREAM(_os_,_core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_, _core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_OFSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_IFSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_FSTREAM(_os_, _core_) //---------------------------------------------------------------------- -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD //---------------------------------------------------------------------- //============================================================================= @@ -255,8 +254,7 @@ class basic_ifstream_dyad basic_ifstream_dyad (const dyad_stream_core& core); basic_ifstream_dyad (); - explicit basic_ifstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::in); + explicit basic_ifstream_dyad (const char* filename, ios_base::openmode mode = ios_base::in); ~basic_ifstream_dyad (); void open (const char* filename, ios_base::openmode mode = ios_base::in); @@ -264,14 +262,12 @@ class basic_ifstream_dyad #if __cplusplus < 201103L bool is_open (); #else - explicit basic_ifstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::in); + explicit basic_ifstream_dyad (const string& filename, ios_base::openmode mode = ios_base::in); basic_ifstream_dyad (const basic_ifstream_dyad&) = delete; basic_ifstream_dyad (basic_ifstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_ifstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::in); + basic_ifstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::in); #endif // c++17 filesystem void open (const string& filename, ios_base::openmode mode = ios_base::in); @@ -289,7 +285,10 @@ class basic_ifstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -392,8 +391,7 @@ basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const _Path& filepath { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = - std::unique_ptr (new basic_ifstream (filepath.c_str (), mode)); + m_stream = std::unique_ptr (new basic_ifstream (filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = std::string{filepath.c_str ()}; } @@ -448,8 +446,7 @@ void basic_ifstream_dyad<_CharT, _Traits>::swap (basic_ifstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_ifstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_ifstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -483,8 +480,7 @@ std::filebuf* basic_ifstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, - _Traits>::get_stream () +std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -515,8 +511,7 @@ class basic_ofstream_dyad basic_ofstream_dyad (const dyad_stream_core& core); basic_ofstream_dyad (); - explicit basic_ofstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::out); + explicit basic_ofstream_dyad (const char* filename, ios_base::openmode mode = ios_base::out); ~basic_ofstream_dyad (); void open (const char* filename, ios_base::openmode mode = ios_base::out); @@ -524,14 +519,12 @@ class basic_ofstream_dyad #if __cplusplus < 201103L bool is_open (); #else - explicit basic_ofstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::out); + explicit basic_ofstream_dyad (const string& filename, ios_base::openmode mode = ios_base::out); basic_ofstream_dyad (const basic_ofstream_dyad&) = delete; basic_ofstream_dyad (basic_ofstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_ofstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::out); + basic_ofstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::out); #endif // c++18 filesystem void open (const string& filename, ios_base::openmode mode = ios_base::out); @@ -549,7 +542,10 @@ class basic_ofstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -590,11 +586,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, m_stream = new basic_ofstream (filename, mode); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -627,11 +622,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, m_core.init (); m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -644,11 +638,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const string& filenam m_core.init (); m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD - && m_core.cmp_canonical_path_prefix (true, (const char* const) filename.c_str ()) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#if DYAD_HAS_STD_FSTREAM_FD + && m_core.cmp_canonical_path_prefix (true, (const char* const)filename.c_str ()) +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = filename; } @@ -667,15 +660,14 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const _Path& filepath std::ios_base::openmode mode) { m_core.init (); - m_stream = - std::unique_ptr (new basic_ofstream ((const char* const) filepath.c_str (), mode)); + m_stream = std::unique_ptr ( + new basic_ofstream ((const char* const)filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename.c_str ()) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filepath.c_str ()}; } @@ -744,8 +736,7 @@ void basic_ofstream_dyad<_CharT, _Traits>::swap (basic_ofstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -753,11 +744,10 @@ void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, } m_stream->open (filename, mode); if ((*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -787,8 +777,7 @@ std::filebuf* basic_ofstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, - _Traits>::get_stream () +std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -820,29 +809,24 @@ class basic_fstream_dyad basic_fstream_dyad (const dyad_stream_core& core); basic_fstream_dyad (); explicit basic_fstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::in - | ios_base::out); + ios_base::openmode mode = ios_base::in | ios_base::out); ~basic_fstream_dyad (); - void open (const char* filename, - ios_base::openmode mode = ios_base::in | ios_base::out); + void open (const char* filename, ios_base::openmode mode = ios_base::in | ios_base::out); #if __cplusplus < 201103L bool is_open (); #else explicit basic_fstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::in - | ios_base::out); + ios_base::openmode mode = ios_base::in | ios_base::out); basic_fstream_dyad (const basic_fstream_dyad&) = delete; basic_fstream_dyad (basic_fstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_fstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::out); + basic_fstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::out); #endif // c++18 filesystem - void open (const string& filename, - ios_base::openmode mode = ios_base::in | ios_base::out); + void open (const string& filename, ios_base::openmode mode = ios_base::in | ios_base::out); bool is_open () const; basic_fstream_dyad& operator= (basic_fstream_dyad&& rhs); @@ -857,7 +841,10 @@ class basic_fstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -899,11 +886,10 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const char* filename, m_stream = new basic_fstream (filename, mode); if ((m_stream != nullptr) && (*m_stream)) { if (m_core.is_dyad_producer () - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); } m_filename = std::string{filename}; @@ -974,8 +960,7 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const _Path& filepath, { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = - std::unique_ptr (new basic_fstream (filepath.c_str (), mode)); + m_stream = std::unique_ptr (new basic_fstream (filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream)) { if (m_core.is_dyad_producer ()) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); @@ -1049,8 +1034,7 @@ void basic_fstream_dyad<_CharT, _Traits>::swap (basic_fstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -1060,11 +1044,10 @@ void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, m_stream->open (filename, mode); if ((*m_stream)) { if (m_core.is_dyad_producer () - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); } m_filename = std::string{filename}; diff --git a/include/dyad/stream/dyad_stream_core.hpp b/include/dyad/stream/dyad_stream_core.hpp index 55e0106b..3ef30416 100644 --- a/include/dyad/stream/dyad_stream_core.hpp +++ b/include/dyad/stream/dyad_stream_core.hpp @@ -17,9 +17,9 @@ #error "no config" #endif +#include #include #include -#include #include @@ -52,12 +52,12 @@ class dyad_stream_core bool chk_initialized () const; bool chk_fsync_write () const; - bool cmp_canonical_path_prefix (bool is_prod, const char* const __restrict__ path); + bool cmp_canonical_path_prefix (bool is_prod, const char *const __restrict__ path); std::string get_upath () const; - int file_lock_exclusive(int fd) const; - int file_lock_shared(int fd) const; - int file_unlock(int fd) const; + int file_lock_exclusive (int fd) const; + int file_lock_shared (int fd) const; + int file_unlock (int fd) const; private: const dyad_ctx *m_ctx; diff --git a/src/dyad/common/dyad_dtl.h b/src/dyad/common/dyad_dtl.h index f9acbfa7..5bcae248 100644 --- a/src/dyad/common/dyad_dtl.h +++ b/src/dyad/common/dyad_dtl.h @@ -11,15 +11,16 @@ extern "C" { #endif - -enum dyad_dtl_mode { DYAD_DTL_UCX = 0, - DYAD_DTL_FLUX_RPC = 1, - DYAD_DTL_DEFAULT = 1, - DYAD_DTL_END = 2 }; +enum dyad_dtl_mode { + DYAD_DTL_UCX = 0, + DYAD_DTL_FLUX_RPC = 1, + DYAD_DTL_DEFAULT = 1, + DYAD_DTL_END = 2 +}; typedef enum dyad_dtl_mode dyad_dtl_mode_t; -static const char* dyad_dtl_mode_name[DYAD_DTL_END+1] __attribute__((unused)) - = {"UCX", "FLUX_RPC", "DTL_UNKNOWN"}; +static const char* dyad_dtl_mode_name[DYAD_DTL_END + 1] + __attribute__ ((unused)) = {"UCX", "FLUX_RPC", "DTL_UNKNOWN"}; enum dyad_dtl_comm_mode { DYAD_COMM_NONE = 0, // Sanity check value for when @@ -30,7 +31,6 @@ enum dyad_dtl_comm_mode { }; typedef enum dyad_dtl_comm_mode dyad_dtl_comm_mode_t; - #define DYAD_DTL_RPC_NAME "dyad.fetch" struct dyad_dtl; diff --git a/src/dyad/common/dyad_logging.h b/src/dyad/common/dyad_logging.h index 0b395662..a2a27841 100644 --- a/src/dyad/common/dyad_logging.h +++ b/src/dyad/common/dyad_logging.h @@ -6,133 +6,141 @@ #error "no config" #endif -#include #include #include +#include + #ifdef __cplusplus extern "C" { #endif -//#define DYAD_NOOP_MACRO do {} while (0) +// #define DYAD_NOOP_MACRO do {} while (0) //============================================================================= #ifdef DYAD_LOGGER_NO_LOG //============================================================================= -#define DYAD_LOGGER_INIT() DYAD_NOOP_MACRO -#define DYAD_LOG_STDERR(...) DYAD_NOOP_MACRO -#define DYAD_LOG_STDOUT(...) DYAD_NOOP_MACRO -#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOGGER_INIT() DYAD_NOOP_MACRO +#define DYAD_LOG_STDERR(...) DYAD_NOOP_MACRO +#define DYAD_LOG_STDOUT(...) DYAD_NOOP_MACRO +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO #define DYAD_LOG_STDOUT_REDIRECT(fpath) DYAD_NOOP_MACRO #define DYAD_LOG_STDERR_REDIRECT(fpath) DYAD_NOOP_MACRO //============================================================================= #else //============================================================================= -#define DYAD_LOG_STDERR(...) fprintf(stderr, __VA_ARGS__); -#define DYAD_LOG_STDOUT(...) fprintf(stdout, __VA_ARGS__); - -#ifdef DYAD_LOGGER_FLUX // FLUX ----------------------------------------------- - #define DYAD_LOGGER_INIT() ; - #define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); - #define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); - - #ifdef DYAD_UTIL_LOGGER - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - #else // DYAD_UTIL_LOGGER - #include - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) flux_log ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) flux_log ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), LOG_INFO, __VA_ARGS__); - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) flux_log_error ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), __VA_ARGS__); - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - #endif // DYAD_UTIL_LOGGER -#elif defined(DYAD_LOGGER_CPP_LOGGER) // CPP_LOGGER --------------------------- - #include - - #define DYAD_LOGGER_NAME "DYAD" - - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); - #elif defined(DYAD_LOGGER_LEVEL_INFO) - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_INFO, DYAD_LOGGER_NAME); - #elif defined(DYAD_LOGGER_LEVEL_WARN) - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_WARN, DYAD_LOGGER_NAME); - #else - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); - #endif - - #define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); - #define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); - - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif -#endif // DYAD_LOGGER_FLUX ---------------------------------------------------- +#define DYAD_LOG_STDERR(...) fprintf (stderr, __VA_ARGS__); +#define DYAD_LOG_STDOUT(...) fprintf (stdout, __VA_ARGS__); + +#ifdef DYAD_LOGGER_FLUX // FLUX ----------------------------------------------- +#define DYAD_LOGGER_INIT() ; +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); + +#ifdef DYAD_UTIL_LOGGER +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR (__VA_ARGS__) +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT (__VA_ARGS__) +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT (__VA_ARGS__) +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR (__VA_ARGS__) +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif +#else // DYAD_UTIL_LOGGER +#include +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + flux_log ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + flux_log ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_INFO, __VA_ARGS__); +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + flux_log_error ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), __VA_ARGS__); +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif +#endif // DYAD_UTIL_LOGGER +#elif defined(DYAD_LOGGER_CPP_LOGGER) // CPP_LOGGER --------------------------- +#include + +#define DYAD_LOGGER_NAME "DYAD" + +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); +#elif defined(DYAD_LOGGER_LEVEL_INFO) +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_INFO, DYAD_LOGGER_NAME); +#elif defined(DYAD_LOGGER_LEVEL_WARN) +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_WARN, DYAD_LOGGER_NAME); +#else +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); +#endif + +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); + +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif +#endif // DYAD_LOGGER_FLUX ---------------------------------------------------- //============================================================================= -#endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG //============================================================================= #ifdef __cplusplus diff --git a/src/dyad/common/dyad_rc.h b/src/dyad/common/dyad_rc.h index 5921141c..aa657254 100644 --- a/src/dyad/common/dyad_rc.h +++ b/src/dyad/common/dyad_rc.h @@ -26,9 +26,9 @@ extern "C" { #endif enum dyad_core_return_codes { - DYAD_RC_OK = 0, // Operation worked correctly - DYAD_RC_SYSFAIL = -1, // Some sys call or C standard - // library call failed + DYAD_RC_OK = 0, // Operation worked correctly + DYAD_RC_SYSFAIL = -1, // Some sys call or C standard + // library call failed // Internal DYAD_RC_NOCTX = -1000, // No DYAD Context found DYAD_RC_BADMETADATA = -1001, // Cannot create/populate a DYAD response @@ -42,27 +42,26 @@ enum dyad_core_return_codes { DYAD_RC_BAD_CLI_PARSE = -1009, // Trying to parse CLI arguments failed DYAD_RC_BADBUF = -1010, // Invalid buffer/pointer passed to function - // FLUX - DYAD_RC_FLUXFAIL = -2000, // Some Flux function failed - DYAD_RC_BADCOMMIT = -2001, // Flux KVS commit didn't work - DYAD_RC_NOTFOUND = -2002, // Flux KVS lookup didn't work - DYAD_RC_BADFETCH = -2003, // Flux KVS commit didn't work - DYAD_RC_BADRPC = -2004, // Flux RPC pack or get didn't work - DYAD_RC_BADPACK = -2005, // JSON packing failed - DYAD_RC_BADUNPACK = -2006, // JSON unpacking failed - DYAD_RC_RPC_FINISHED = -2007, // The Flux RPC responded with ENODATA (i.e., - // end of stream) sooner than expected + DYAD_RC_FLUXFAIL = -2000, // Some Flux function failed + DYAD_RC_BADCOMMIT = -2001, // Flux KVS commit didn't work + DYAD_RC_NOTFOUND = -2002, // Flux KVS lookup didn't work + DYAD_RC_BADFETCH = -2003, // Flux KVS commit didn't work + DYAD_RC_BADRPC = -2004, // Flux RPC pack or get didn't work + DYAD_RC_BADPACK = -2005, // JSON packing failed + DYAD_RC_BADUNPACK = -2006, // JSON unpacking failed + DYAD_RC_RPC_FINISHED = -2007, // The Flux RPC responded with ENODATA (i.e., + // end of stream) sooner than expected - //UCX - DYAD_RC_UCXINIT_FAIL = -3001, // UCX initialization failed - DYAD_RC_UCXWAIT_FAIL = -3002, // UCX wait (either custom or - // 'ucp_worker_wait') failed - DYAD_RC_UCXEP_FAIL = -3003, // An operation on a ucp_ep_h failed - DYAD_RC_UCXCOMM_FAIL = -3004, // UCX communication routine failed - DYAD_RC_UCXMMAP_FAIL = -3005, // Failed to perform operations with ucp_mem_map - DYAD_RC_UCXRKEY_PACK_FAILED = -3006, // Failed to perform operations with ucp_mem_map - DYAD_RC_UCXRKEY_UNPACK_FAILED = -3007, // Failed to perform operations with ucp_mem_map + // UCX + DYAD_RC_UCXINIT_FAIL = -3001, // UCX initialization failed + DYAD_RC_UCXWAIT_FAIL = -3002, // UCX wait (either custom or + // 'ucp_worker_wait') failed + DYAD_RC_UCXEP_FAIL = -3003, // An operation on a ucp_ep_h failed + DYAD_RC_UCXCOMM_FAIL = -3004, // UCX communication routine failed + DYAD_RC_UCXMMAP_FAIL = -3005, // Failed to perform operations with ucp_mem_map + DYAD_RC_UCXRKEY_PACK_FAILED = -3006, // Failed to perform operations with ucp_mem_map + DYAD_RC_UCXRKEY_UNPACK_FAILED = -3007, // Failed to perform operations with ucp_mem_map }; diff --git a/src/dyad/common/dyad_structures.h b/src/dyad/common/dyad_structures.h index 0f603bc1..048538e8 100644 --- a/src/dyad/common/dyad_structures.h +++ b/src/dyad/common/dyad_structures.h @@ -14,7 +14,6 @@ #include #endif - #ifdef __cplusplus extern "C" { #endif @@ -24,21 +23,21 @@ extern "C" { */ struct dyad_ctx { // Internal - void* h; // the Flux handle for DYAD - struct dyad_dtl* dtl_handle; // Opaque handle to DTL info - const char* fname; // Used to track which file is getting processed. - bool use_fs_locks; // Used to track if fs locks should be used. - char* prod_real_path; // producer managed real path - char* cons_real_path; // consumer managed real path - uint32_t prod_managed_len; // length of producer path managed by DYAD - uint32_t cons_managed_len; // length of consumer path managed by DYAD - uint32_t prod_real_len; // length of producer managed real path - uint32_t cons_real_len; // length of consumer managed real path - uint32_t prod_managed_hash; // hash of producer path managed by DYAD - uint32_t cons_managed_hash; // hash of consumer path managed by DYAD - uint32_t prod_real_hash; // hash of producer managed real path - uint32_t cons_real_hash; // hash of consumer managed real path - uint32_t delim_len; // length of path delimiter + void* h; // the Flux handle for DYAD + struct dyad_dtl* dtl_handle; // Opaque handle to DTL info + const char* fname; // Used to track which file is getting processed. + bool use_fs_locks; // Used to track if fs locks should be used. + char* prod_real_path; // producer managed real path + char* cons_real_path; // consumer managed real path + uint32_t prod_managed_len; // length of producer path managed by DYAD + uint32_t cons_managed_len; // length of consumer path managed by DYAD + uint32_t prod_real_len; // length of producer managed real path + uint32_t cons_real_len; // length of consumer managed real path + uint32_t prod_managed_hash; // hash of producer path managed by DYAD + uint32_t cons_managed_hash; // hash of consumer path managed by DYAD + uint32_t prod_real_hash; // hash of producer managed real path + uint32_t cons_real_hash; // hash of consumer managed real path + uint32_t delim_len; // length of path delimiter // User Facing bool debug; // if true, perform debug logging bool check; // if true, perform some check logging @@ -61,7 +60,6 @@ struct dyad_ctx { typedef struct dyad_ctx dyad_ctx_t; typedef void* ucx_ep_cache_h; - #ifdef __cplusplus } #endif diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c index e51c153a..63cea19d 100644 --- a/src/dyad/core/dyad_core.c +++ b/src/dyad/core/dyad_core.c @@ -4,19 +4,11 @@ #error "no config" #endif -#include -#include -#include -#include -#include -#include -#include -#include #include +#include #include #include #include -#include #ifdef __cplusplus #include @@ -27,16 +19,22 @@ #include #endif - - +#include +#include +#include +#include +#include +#include +#include +#include DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, - char* restrict path_key, - const size_t len, - const uint32_t depth, - const uint32_t width) + char* restrict path_key, + const size_t len, + const uint32_t depth, + const uint32_t width) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); static const uint32_t seeds[10] = {104677u, 104681u, 104683u, 104693u, 104701u, 104707u, 104711u, 104717u, 104723u, 104729u}; @@ -45,16 +43,16 @@ DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, size_t cx = 0ul; int n = 0; if (str == NULL || path_key == NULL || len == 0ul) { - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return -1; } size_t str_len = strlen (str); if (str_len == 0ul) { - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return -1; } const char* str_long = str; - + path_key[0] = '\0'; // Just append the string so that it can be as large as 128 bytes. @@ -74,22 +72,22 @@ DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, n = snprintf (path_key + cx, len - cx, "%x.", bin); cx += n; if (cx >= len || n < 0) { - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return -1; } } n = snprintf (path_key + cx, len - cx, "%s", str); // FIXME: cx + n >= len fails for str_len > 256 if (n < 0) { - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return -1; } DYAD_C_FUNCTION_UPDATE_STR ("path_key", path_key); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return 0; } -static void future_cleanup_cb (flux_future_t *f, void *arg) +static void future_cleanup_cb (flux_future_t* f, void* arg) { if (flux_future_get (f, NULL) < 0) { DYAD_LOG_STDERR ("future_cleanup: future error detected with.%s", ""); @@ -100,12 +98,12 @@ static void future_cleanup_cb (flux_future_t *f, void *arg) DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* restrict ctx, flux_kvs_txn_t* restrict txn) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); flux_future_t* f = NULL; dyad_rc_t rc = DYAD_RC_OK; DYAD_LOG_INFO (ctx, "Committing transaction to KVS"); // Commit the transaction to the Flux KVS - f = flux_kvs_commit ((flux_t*) ctx->h, ctx->kvs_namespace, 0, txn); + f = flux_kvs_commit ((flux_t*)ctx->h, ctx->kvs_namespace, 0, txn); // If the commit failed, log an error and return DYAD_BADCOMMIT if (f == NULL) { DYAD_LOG_ERROR (ctx, "Could not commit transaction to Flux KVS"); @@ -125,14 +123,14 @@ DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* restrict ctx, } rc = DYAD_RC_OK; kvs_commit_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux (const dyad_ctx_t* restrict ctx, const char* restrict upath) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); dyad_rc_t rc = DYAD_RC_OK; @@ -173,16 +171,16 @@ publish_done:; if (txn != NULL) { flux_kvs_txn_destroy (txn); } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, const char* restrict fname) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); dyad_rc_t rc = DYAD_RC_OK; - char upath[PATH_MAX+1] = {'\0'}; + char upath[PATH_MAX + 1] = {'\0'}; #if 0 if (fname == NULL || strlen (fname) > PATH_MAX) { rc = DYAD_RC_SYSFAIL; @@ -193,9 +191,9 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, const char* r // must be a valid string (!NULL). ctx->delim_len is verified to be greater // than 0 during initialization. if (ctx->relative_to_managed_path && - //(strlen (fname) > 0ul) && // checked where get_path() was - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the prod_managed_path + //(strlen (fname) > 0ul) && // checked where get_path() was + (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the prod_managed_path memcpy (upath, fname, strlen (fname)); } else if (!cmp_canonical_path_prefix (ctx, true, fname, upath, PATH_MAX)) { // Extract the path to the file specified by fname relative to the @@ -221,12 +219,11 @@ commit_done:; if (rc == DYAD_RC_OK && (ctx && ctx->check)) { setenv (DYAD_CHECK_ENV, "ok", 1); } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static void print_mdata (const dyad_ctx_t* restrict ctx, - const dyad_metadata_t* restrict mdata) +static void print_mdata (const dyad_ctx_t* restrict ctx, const dyad_metadata_t* restrict mdata) { if (mdata == NULL) { DYAD_LOG_INFO (ctx, "Cannot print a NULL metadata object!"); @@ -238,18 +235,19 @@ static void print_mdata (const dyad_ctx_t* restrict ctx, } DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* restrict ctx, - const char* restrict topic, - const char* restrict upath, - bool should_wait, - dyad_metadata_t** restrict mdata) + const char* restrict topic, + const char* restrict upath, + bool should_wait, + dyad_metadata_t** restrict mdata) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); dyad_rc_t rc = DYAD_RC_OK; int kvs_lookup_flags = 0; flux_future_t* f = NULL; if (mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Metadata double pointer is NULL. " \ + DYAD_LOG_ERROR (ctx, + "Metadata double pointer is NULL. " "Cannot correctly create metadata object"); rc = DYAD_RC_NOTFOUND; goto kvs_read_end; @@ -260,7 +258,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* restrict ctx, if (should_wait) kvs_lookup_flags = FLUX_KVS_WAITCREATE; DYAD_LOG_INFO (ctx, "Retrieving information from KVS under the key %s", topic); - f = flux_kvs_lookup ((flux_t*) ctx->h, ctx->kvs_namespace, kvs_lookup_flags, topic); + f = flux_kvs_lookup ((flux_t*)ctx->h, ctx->kvs_namespace, kvs_lookup_flags, topic); // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP if (f == NULL) { DYAD_LOG_ERROR (ctx, "KVS lookup failed!\n"); @@ -309,22 +307,20 @@ kvs_read_end:; flux_future_destroy (f); f = NULL; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } - - DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata (const dyad_ctx_t* restrict ctx, const char* restrict fname, const char* restrict upath, dyad_metadata_t** restrict mdata) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); dyad_rc_t rc = DYAD_RC_OK; const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; + char topic[PATH_MAX + 1] = {'\0'}; *mdata = NULL; #if 0 if (fname == NULL || upath == NULL || strlen (fname) == 0ul || strlen (upath) == 0ul) { @@ -357,9 +353,11 @@ DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata (const dyad_ctx_t* restrict ct DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", (*mdata)->owner_rank); DYAD_C_FUNCTION_UPDATE_INT ("node_idx", ctx->node_idx); if (((*mdata)->owner_rank / ctx->service_mux) == ctx->node_idx) { - DYAD_LOG_INFO (ctx, \ - "Either shared-storage is indicated or the producer rank (%u) is the" \ - " same as the consumer rank (%u)", (*mdata)->owner_rank, ctx->rank); + DYAD_LOG_INFO (ctx, + "Either shared-storage is indicated or the producer rank (%u) is the" + " same as the consumer rank (%u)", + (*mdata)->owner_rank, + ctx->rank); if (mdata != NULL && *mdata != NULL) { dyad_free_metadata (mdata); } @@ -371,16 +369,16 @@ DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata (const dyad_ctx_t* restrict ct rc = DYAD_RC_OK; fetch_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, - const dyad_metadata_t* restrict mdata, - char** restrict file_data, - size_t* restrict file_len) + const dyad_metadata_t* restrict mdata, + char** restrict file_data, + size_t* restrict file_len) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; flux_future_t* f = NULL; json_t* rpc_payload = NULL; @@ -389,12 +387,13 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, DYAD_C_FUNCTION_UPDATE_STR ("fpath", mdata->fpath); rc = ctx->dtl_handle->rpc_pack (ctx, mdata->fpath, mdata->owner_rank, &rpc_payload); if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot create JSON payload for Flux RPC to " \ - "DYAD module\n"); + DYAD_LOG_ERROR (ctx, + "Cannot create JSON payload for Flux RPC to " + "DYAD module\n"); goto get_done; } DYAD_LOG_INFO (ctx, "Sending payload for RPC to DYAD module"); - f = flux_rpc_pack ((flux_t*) ctx->h, + f = flux_rpc_pack ((flux_t*)ctx->h, DYAD_DTL_RPC_NAME, mdata->owner_rank, FLUX_RPC_STREAMING, @@ -414,10 +413,10 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, DYAD_LOG_INFO (ctx, "Establish DTL connection with DYAD module"); rc = ctx->dtl_handle->establish_connection (ctx); if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, \ - "Cannot establish connection with DYAD module on broker " \ - "%u\n", \ - mdata->owner_rank); + DYAD_LOG_ERROR (ctx, + "Cannot establish connection with DYAD module on broker " + "%u\n", + mdata->owner_rank); goto get_done; } DYAD_LOG_INFO (ctx, "Receive file data via DTL"); @@ -446,16 +445,16 @@ get_done:; DYAD_LOG_INFO (ctx, "Wait for end-of-stream message from module (current RC = %d)\n", rc); if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { if (!(flux_rpc_get (f, NULL) < 0 && errno == ENODATA)) { - DYAD_LOG_ERROR (ctx, \ - "An error occured at end of getting data! Either the " \ - "module sent too many responses, or the module " \ - "failed with a bad error (errno = %d)\n", \ + DYAD_LOG_ERROR (ctx, + "An error occured at end of getting data! Either the " + "module sent too many responses, or the module " + "failed with a bad error (errno = %d)\n", errno); rc = DYAD_RC_BADRPC; } } #ifdef DYAD_ENABLE_UCX_RMA - ctx->dtl_handle->get_buffer(ctx, 0, (void**)file_data); + ctx->dtl_handle->get_buffer (ctx, 0, (void**)file_data); ssize_t read_len = 0l; memcpy (&read_len, *file_data, sizeof (read_len)); if (read_len < 0l) { @@ -463,23 +462,24 @@ get_done:; DYAD_LOG_DEBUG (ctx, "Not able to read from %s file", mdata->fpath); rc = DYAD_RC_BADFIO; } else { - *file_len = (size_t) read_len; + *file_len = (size_t)read_len; } *file_data = ((char*)*file_data) + sizeof (read_len); DYAD_LOG_INFO (ctx, "Read %zd bytes from %s file", *file_len, mdata->fpath); #endif DYAD_LOG_INFO (ctx, "Destroy the Flux future for the RPC\n"); flux_future_destroy (f); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } DYAD_CORE_FUNC_MODS dyad_rc_t dyad_cons_store (const dyad_ctx_t* restrict ctx, const dyad_metadata_t* restrict mdata, - int fd, const size_t data_len, + int fd, + const size_t data_len, char* restrict file_data) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); dyad_rc_t rc = DYAD_RC_OK; const char* odir = NULL; @@ -522,14 +522,14 @@ pull_done:; // If "check" is set and the operation was successful, set the // DYAD_CHECK_ENV environment variable to "ok" if (rc == DYAD_RC_OK && (ctx && ctx->check)) - setenv(DYAD_CHECK_ENV, "ok", 1); - DYAD_C_FUNCTION_END(); + setenv (DYAD_CHECK_ENV, "ok", 1); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_produce (dyad_ctx_t* restrict ctx, const char* restrict fname) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ctx->fname = fname; DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); DYAD_LOG_DEBUG (ctx, "Executing dyad_produce"); @@ -537,15 +537,16 @@ dyad_rc_t dyad_produce (dyad_ctx_t* restrict ctx, const char* restrict fname) // If the context is not defined, then it is not valid. // So, return DYAD_NOCTX if (!ctx || !ctx->h) { - DYAD_LOG_ERROR(ctx, "No CTX found in dyad_produce"); + DYAD_LOG_ERROR (ctx, "No CTX found in dyad_produce"); rc = DYAD_RC_NOCTX; goto produce_done; } // If the producer-managed path is NULL or empty, then the context is not // valid for a producer operation. So, return DYAD_BADMANAGEDPATH if (ctx->prod_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, "No or empty producer managed path was found %s", \ - ctx->prod_managed_path); + DYAD_LOG_ERROR (ctx, + "No or empty producer managed path was found %s", + ctx->prod_managed_path); rc = DYAD_RC_BADMANAGEDPATH; goto produce_done; } @@ -553,7 +554,7 @@ dyad_rc_t dyad_produce (dyad_ctx_t* restrict ctx, const char* restrict fname) // the producer operation rc = dyad_commit (ctx, fname); produce_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -565,7 +566,7 @@ dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, bool should_wait, dyad_metadata_t** restrict mdata) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); DYAD_C_FUNCTION_UPDATE_INT ("should_wait", should_wait); dyad_rc_t rc = DYAD_RC_OK; @@ -577,7 +578,7 @@ dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, } #endif const size_t fname_len = strlen (fname); - char upath[PATH_MAX+1] = {'\0'}; + char upath[PATH_MAX + 1] = {'\0'}; DYAD_LOG_INFO (ctx, "Obtaining file path relative to consumer directory: %s", upath); @@ -585,15 +586,15 @@ dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, rc = DYAD_RC_BADFIO; goto get_metadata_done; } - if (ctx->relative_to_managed_path && - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the cons_managed_path + if (ctx->relative_to_managed_path + && (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the cons_managed_path memcpy (upath, fname, fname_len); } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { // Extract the path to the file specified by fname relative to the // producer-managed path // This relative path will be stored in upath - //DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); // NOTE: This is different from what dyad_fetch/commit returns, // which is DYAD_RC_OK such that dyad does not interfere accesses on // non-managed directories. @@ -608,8 +609,9 @@ dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, if (fd != -1) { close (fd); if (mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Metadata double pointer is NULL. " \ - "Cannot correctly create metadata object"); + DYAD_LOG_ERROR (ctx, + "Metadata double pointer is NULL. " + "Cannot correctly create metadata object"); rc = DYAD_RC_NOTFOUND; goto get_metadata_done; } @@ -637,7 +639,7 @@ dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, } const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; + char topic[PATH_MAX + 1] = {'\0'}; DYAD_LOG_INFO (ctx, "Generating KVS key: %s", topic); gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); rc = dyad_kvs_read (ctx, topic, upath, should_wait, mdata); @@ -652,13 +654,13 @@ get_metadata_done:; dyad_free_metadata (mdata); } ctx->reenter = true; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_free_metadata (dyad_metadata_t** mdata) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); if (mdata == NULL || *mdata == NULL) { return DYAD_RC_OK; } @@ -666,13 +668,13 @@ dyad_rc_t dyad_free_metadata (dyad_metadata_t** mdata) free ((*mdata)->fpath); free (*mdata); *mdata = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); dyad_rc_t rc = DYAD_RC_OK; int lock_fd = -1, io_fd = -1; @@ -681,7 +683,7 @@ dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) size_t data_len = 0ul; dyad_metadata_t* mdata = NULL; struct flock exclusive_lock; - char upath[PATH_MAX+1] = {'\0'}; + char upath[PATH_MAX + 1] = {'\0'}; // If the context is not defined, then it is not valid. // So, return DYAD_NOCTX @@ -696,15 +698,15 @@ dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) goto consume_close; } - if (ctx->relative_to_managed_path && (strlen (fname) > 0ul) && - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the cons_managed_path + if (ctx->relative_to_managed_path && (strlen (fname) > 0ul) + && (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the cons_managed_path memcpy (upath, fname, strlen (fname)); } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { // Extract the path to the file specified by fname relative to the // consumer-managed path // This relative path will be stored in upath - //DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); rc = DYAD_RC_OK; goto consume_close; } @@ -737,8 +739,13 @@ dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) } } else { if (file_size <= 0) { - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with lock_fd %d) is not fetched yet", \ - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] File (%s with lock_fd %d) is not fetched yet", + ctx->node_idx, + ctx->rank, + ctx->pid, + fname, + lock_fd); // Call dyad_fetch to get (and possibly wait on) // data from the Flux KVS rc = dyad_fetch_metadata (ctx, fname, upath, &mdata); @@ -772,7 +779,9 @@ dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) io_fd = open (fname, O_WRONLY); DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); if (io_fd == -1) { - DYAD_LOG_ERROR (ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", fname); + DYAD_LOG_ERROR (ctx, + "Cannot open file (%s) in write mode for dyad_consume!\n", + fname); rc = DYAD_RC_BADFIO; goto consume_close; } @@ -810,14 +819,15 @@ consume_done:; // Set reenter to true to allow additional intercepting consume_close:; ctx->reenter = true; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, +dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, + const char* fname, const dyad_metadata_t* restrict mdata) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); dyad_rc_t rc = DYAD_RC_OK; int lock_fd = -1, io_fd = -1; @@ -859,8 +869,13 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, goto consume_close; } if ((file_size = get_file_size (lock_fd)) <= 0) { - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", \ - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", + ctx->node_idx, + ctx->rank, + ctx->pid, + fname, + lock_fd); // Call dyad_get_data to dispatch a RPC to the producer's Flux broker // and retrieve the data associated with the file @@ -910,14 +925,14 @@ consume_done:; consume_close:; // Set reenter to true to allow additional intercepting ctx->reenter = true; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } #if DYAD_SYNC_DIR int dyad_sync_directory (dyad_ctx_t* restrict ctx, const char* restrict path) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("path", path); // Flush new directory entry https://lwn.net/Articles/457671/ char path_copy[PATH_MAX + 1] = {'\0'}; @@ -949,7 +964,7 @@ int dyad_sync_directory (dyad_ctx_t* restrict ctx, const char* restrict path) } if (ctx != NULL) ctx->reenter = reenter; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } #endif diff --git a/src/dyad/core/dyad_core.h b/src/dyad/core/dyad_core.h index b58577ca..ff8c0133 100644 --- a/src/dyad/core/dyad_core.h +++ b/src/dyad/core/dyad_core.h @@ -7,8 +7,6 @@ #error "no config" #endif -#include -#include #include #include @@ -21,6 +19,8 @@ #include #endif +#include +#include #ifdef __cplusplus extern "C" { @@ -41,16 +41,16 @@ typedef struct dyad_metadata dyad_metadata_t; // Debug message #ifndef DPRINTF #if VA_OPT_SUPPORTED -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ - if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf (stderr, (fmt) __VA_OPT__(,) __VA_ARGS__); \ +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ + if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ + fprintf (stderr, (fmt)__VA_OPT__ (, ) __VA_ARGS__); \ } while (0) #else -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf (stderr, (fmt), ##__VA_ARGS__); \ + fprintf (stderr, (fmt), ##__VA_ARGS__); \ } while (0) #endif #endif // DPRINTF @@ -67,7 +67,6 @@ typedef struct dyad_metadata dyad_metadata_t; #define IPRINTF(curr_dyad_ctx, fmt, ...) #endif // DYAD_FULL_DEBUG - /** * @brief Wrapper function that performs all the common tasks needed * of a producer @@ -114,28 +113,29 @@ DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume (dyad_ctx_t* ctx, con * * @return An error code from dyad_rc.h */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* ctx, const char* fname, - const dyad_metadata_t* mdata); - +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t +dyad_consume_w_metadata (dyad_ctx_t* ctx, const char* fname, const dyad_metadata_t* mdata); /** * Private Function definitions */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* ctx, const dyad_metadata_t* mdata, - char** file_data, - size_t* file_len); +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* ctx, + const dyad_metadata_t* mdata, + char** file_data, + size_t* file_len); DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* ctx, const char* fname); -DYAD_DLL_EXPORTED int gen_path_key (const char* str, char* path_key, - const size_t len, - const uint32_t depth, - const uint32_t width); +DYAD_DLL_EXPORTED int gen_path_key (const char* str, + char* path_key, + const size_t len, + const uint32_t depth, + const uint32_t width); DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* ctx, - const char* topic, - const char* upath, - bool should_wait, - dyad_metadata_t** mdata); + const char* topic, + const char* upath, + bool should_wait, + dyad_metadata_t** mdata); #if DYAD_SYNC_DIR DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory (dyad_ctx_t* ctx, const char* path); diff --git a/src/dyad/core/dyad_ctx.c b/src/dyad/core/dyad_ctx.c index 376d22aa..223aa92d 100644 --- a/src/dyad/core/dyad_ctx.c +++ b/src/dyad/core/dyad_ctx.c @@ -4,13 +4,7 @@ #error "no config" #endif -#include -#include -#include -#include -#include -#include -#include +#include #include #ifdef __cplusplus @@ -26,7 +20,13 @@ #include #endif -#include +#include +#include +#include +#include +#include +#include +#include // Note: // To ensure we don't have multiple initialization, we need the following: @@ -289,7 +289,9 @@ dyad_rc_t dyad_init (bool debug, } ctx->relative_to_managed_path = relative_to_managed_path; - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: relative_to_managed_path %s", ctx->relative_to_managed_path ? "true" : "false"); + DYAD_LOG_INFO (ctx, + "DYAD_CORE INIT: relative_to_managed_path %s", + ctx->relative_to_managed_path ? "true" : "false"); DYAD_C_FUNCTION_UPDATE_STR ("prod_managed_path", ctx->prod_managed_path); DYAD_C_FUNCTION_UPDATE_STR ("cons_managed_path", ctx->cons_managed_path); @@ -803,7 +805,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize () { DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; - //DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); + // DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); if (ctx == NULL) { rc = DYAD_RC_OK; goto finalize_region_finish; diff --git a/src/dyad/core/dyad_ctx.h b/src/dyad/core/dyad_ctx.h index e057f319..b1f5a5ad 100644 --- a/src/dyad/core/dyad_ctx.h +++ b/src/dyad/core/dyad_ctx.h @@ -7,8 +7,8 @@ #error "no config" #endif -#include #include +#include #include #ifdef __cplusplus @@ -16,8 +16,7 @@ extern "C" { #endif DYAD_DLL_EXPORTED dyad_ctx_t* dyad_ctx_get (); -DYAD_DLL_EXPORTED void dyad_ctx_init (dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle); +DYAD_DLL_EXPORTED void dyad_ctx_init (dyad_dtl_comm_mode_t dtl_comm_mode, void* flux_handle); DYAD_DLL_EXPORTED void dyad_ctx_fini (); /** @@ -59,7 +58,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_init (bool debug, const char* prod_managed_path, const char* cons_managed_path, bool relative_to_managed_path, - const char* dtl_mode_str, + const char* dtl_mode_str, const dyad_dtl_comm_mode_t dtl_comm_mode, void* flux_handle); diff --git a/src/dyad/dtl/dyad_dtl_api.c b/src/dyad/dtl/dyad_dtl_api.c index bc35a7ce..268af4bd 100644 --- a/src/dyad/dtl/dyad_dtl_api.c +++ b/src/dyad/dtl/dyad_dtl_api.c @@ -4,27 +4,27 @@ #error "no config" #endif -#include -#include #include #include +#include +#include #if DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA #include "ucx_dtl.h" -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, dyad_dtl_mode_t mode, dyad_dtl_comm_mode_t comm_mode, bool debug) { - DYAD_C_FUNCTION_START(); - DYAD_LOG_DEBUG(ctx, "Initializing DTL ..."); + DYAD_C_FUNCTION_START (); + DYAD_LOG_DEBUG (ctx, "Initializing DTL ..."); dyad_rc_t rc = DYAD_RC_OK; ctx->dtl_handle = malloc (sizeof (struct dyad_dtl)); if (ctx->dtl_handle == NULL) { rc = DYAD_RC_SYSFAIL; - DYAD_LOG_ERROR(ctx, "Cannot allocate Memory"); + DYAD_LOG_ERROR (ctx, "Cannot allocate Memory"); goto dtl_init_done; } ctx->dtl_handle->mode = mode; @@ -36,9 +36,9 @@ dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, goto dtl_init_done; } } else if (mode == DYAD_DTL_FLUX_RPC) { -#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if (mode == DYAD_DTL_FLUX_RPC) { -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA rc = dyad_dtl_flux_init (ctx, mode, comm_mode, debug); if (DYAD_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, "dyad_dtl_flux_init initialization failed rc %d", rc); @@ -46,19 +46,21 @@ dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, } } else { rc = DYAD_RC_BADDTLMODE; - DYAD_LOG_ERROR (ctx, "dyad_dtl_flux_init initialization failed with incorrect mode %d", mode); + DYAD_LOG_ERROR (ctx, + "dyad_dtl_flux_init initialization failed with incorrect mode %d", + mode); goto dtl_init_done; } rc = DYAD_RC_OK; DYAD_LOG_DEBUG (ctx, "Finished dyad_dtl_init successfully"); dtl_init_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL) { // We should only reach this line if the user has passed @@ -77,9 +79,9 @@ dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) } } } else if ((ctx->dtl_handle)->mode == DYAD_DTL_FLUX_RPC) { -#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if ((ctx->dtl_handle)->mode == DYAD_DTL_FLUX_RPC) { -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if ((ctx->dtl_handle)->private_dtl.flux_dtl_handle != NULL) { rc = dyad_dtl_flux_finalize (ctx); if (DYAD_IS_ERROR (rc)) { @@ -95,6 +97,6 @@ dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) dtl_finalize_done:; free (ctx->dtl_handle); ctx->dtl_handle = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/dyad_dtl_api.h b/src/dyad/dtl/dyad_dtl_api.h index 92081bcf..51528e84 100644 --- a/src/dyad/dtl/dyad_dtl_api.h +++ b/src/dyad/dtl/dyad_dtl_api.h @@ -7,20 +7,23 @@ #error "no config" #endif +#include #include #include -#include -#include -#include -#include #ifdef __cplusplus #include - -extern "C" { #else #include #endif +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Forward declarations of DTL contexts for the underlying implementations struct dyad_dtl_ucx; struct dyad_dtl_flux; @@ -29,16 +32,16 @@ struct dyad_dtl_flux; union dyad_dtl_private { struct dyad_dtl_ucx* ucx_dtl_handle; struct dyad_dtl_flux* flux_dtl_handle; -} __attribute__((aligned(16))); +} __attribute__ ((aligned (16))); typedef union dyad_dtl_private dyad_dtl_private_t; struct dyad_dtl { dyad_dtl_private_t private_dtl; dyad_dtl_mode_t mode; dyad_rc_t (*rpc_pack) (const dyad_ctx_t* ctx, - const char* upath, + const char* upath, uint32_t producer_rank, - json_t** packed_obj); + json_t** packed_obj); dyad_rc_t (*rpc_unpack) (const dyad_ctx_t* ctx, const flux_msg_t* packed_obj, char** upath); dyad_rc_t (*rpc_respond) (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg); dyad_rc_t (*rpc_recv_response) (const dyad_ctx_t* ctx, flux_future_t* f); @@ -48,7 +51,7 @@ struct dyad_dtl { dyad_rc_t (*send) (const dyad_ctx_t* ctx, void* buf, size_t buflen); dyad_rc_t (*recv) (const dyad_ctx_t* ctx, void** buf, size_t* buflen); dyad_rc_t (*close_connection) (const dyad_ctx_t* ctx); -} __attribute__((aligned(256))); +} __attribute__ ((aligned (256))); typedef struct dyad_dtl dyad_dtl_t; dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, diff --git a/src/dyad/dtl/flux_dtl.c b/src/dyad/dtl/flux_dtl.c index 80c7aeda..a959a2eb 100644 --- a/src/dyad/dtl/flux_dtl.c +++ b/src/dyad/dtl/flux_dtl.c @@ -4,10 +4,11 @@ #error "no config" #endif -#include +#include // sysconf + #include #include -#include // sysconf +#include dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, dyad_dtl_mode_t mode, @@ -15,14 +16,14 @@ dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, bool debug) { dyad_rc_t rc = DYAD_RC_OK; - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ctx->dtl_handle->private_dtl.flux_dtl_handle = malloc (sizeof (struct dyad_dtl_flux)); if (ctx->dtl_handle->private_dtl.flux_dtl_handle == NULL) { DYAD_LOG_ERROR (ctx, "Cannot allocate the Flux DTL handle"); rc = DYAD_RC_SYSFAIL; goto dtl_flux_init_region_finish; } - ctx->dtl_handle->private_dtl.flux_dtl_handle->h = (flux_t*) ctx->h; + ctx->dtl_handle->private_dtl.flux_dtl_handle->h = (flux_t*)ctx->h; ctx->dtl_handle->private_dtl.flux_dtl_handle->comm_mode = comm_mode; ctx->dtl_handle->private_dtl.flux_dtl_handle->debug = debug; ctx->dtl_handle->private_dtl.flux_dtl_handle->f = NULL; @@ -40,7 +41,7 @@ dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, ctx->dtl_handle->close_connection = dyad_dtl_flux_close_connection; dtl_flux_init_region_finish: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -49,7 +50,7 @@ dyad_rc_t dyad_dtl_flux_rpc_pack (const dyad_ctx_t* ctx, uint32_t producer_rank, json_t** restrict packed_obj) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); DYAD_C_FUNCTION_UPDATE_INT ("producer_rank", producer_rank); dyad_rc_t rc = DYAD_RC_OK; @@ -60,19 +61,18 @@ dyad_rc_t dyad_dtl_flux_rpc_pack (const dyad_ctx_t* ctx, goto dtl_flux_rpc_pack; } dtl_flux_rpc_pack: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); int rc = 0; dyad_rc_t dyad_rc = DYAD_RC_OK; rc = flux_request_unpack (msg, NULL, "{s:s}", "upath", upath); if (FLUX_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, - "Could not unpack Flux message from consumer"); + DYAD_LOG_ERROR (ctx, "Could not unpack Flux message from consumer"); // TODO create new RC for this dyad_rc = DYAD_RC_BADUNPACK; goto dtl_flux_rpc_unpack_region_finish; @@ -81,28 +81,28 @@ dyad_rc_t dyad_dtl_flux_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg dyad_rc = DYAD_RC_OK; DYAD_C_FUNCTION_UPDATE_STR ("upath", *upath); dtl_flux_rpc_unpack_region_finish: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_rpc_respond (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_rpc_recv_response (const dyad_ctx_t* ctx, flux_future_t* f) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ctx->dtl_handle->private_dtl.flux_dtl_handle->f = f; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf != NULL) { @@ -116,7 +116,7 @@ dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, voi goto flux_get_buf_done; } #else - rc = posix_memalign (data_buf, sysconf(_SC_PAGESIZE), data_size); + rc = posix_memalign (data_buf, sysconf (_SC_PAGESIZE), data_size); if (rc != 0 || *data_buf == NULL) { rc = DYAD_RC_SYSFAIL; goto flux_get_buf_done; @@ -125,13 +125,13 @@ dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, voi rc = DYAD_RC_OK; flux_get_buf_done: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_return_buffer (const dyad_ctx_t* ctx, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf == NULL) { rc = DYAD_RC_BADBUF; @@ -141,49 +141,47 @@ dyad_rc_t dyad_dtl_flux_return_buffer (const dyad_ctx_t* ctx, void** data_buf) rc = DYAD_RC_OK; flux_ret_buf_done: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_establish_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t dyad_rc = DYAD_RC_OK; int rc = 0; - DYAD_LOG_INFO (ctx, - "Send data to consumer using a Flux RPC response"); + DYAD_LOG_INFO (ctx, "Send data to consumer using a Flux RPC response"); rc = flux_respond_raw (ctx->dtl_handle->private_dtl.flux_dtl_handle->h, ctx->dtl_handle->private_dtl.flux_dtl_handle->msg, buf, (int)buflen); if (FLUX_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, - "Could not send Flux RPC response containing file " - "contents"); + "Could not send Flux RPC response containing file " + "contents"); dyad_rc = DYAD_RC_FLUXFAIL; goto dtl_flux_send_region_finish; } if (ctx->dtl_handle->private_dtl.flux_dtl_handle->debug) { - DYAD_LOG_INFO (ctx, - "Successfully sent file contents to consumer"); + DYAD_LOG_INFO (ctx, "Successfully sent file contents to consumer"); } dyad_rc = DYAD_RC_OK; dtl_flux_send_region_finish: DYAD_C_FUNCTION_UPDATE_INT ("buflen", buflen); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); int rc = 0; dyad_rc_t dyad_rc = DYAD_RC_OK; errno = 0; @@ -206,7 +204,7 @@ dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) dyad_rc = DYAD_RC_BADRPC; goto finish_recv; } - *buflen = (size_t) tmp_buflen; + *buflen = (size_t)tmp_buflen; dyad_rc = ctx->dtl_handle->get_buffer (ctx, *buflen, buf); if (DYAD_IS_ERROR (dyad_rc)) { *buf = NULL; @@ -216,27 +214,27 @@ dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) memcpy (*buf, tmp_buf, *buflen); dyad_rc = DYAD_RC_OK; finish_recv: - if (dtl_handle->f != NULL) + if (dtl_handle->f != NULL) flux_future_reset (dtl_handle->f); DYAD_C_FUNCTION_UPDATE_INT ("tmp_buflen", tmp_buflen); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_close_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); if (ctx->dtl_handle->private_dtl.flux_dtl_handle->f != NULL) ctx->dtl_handle->private_dtl.flux_dtl_handle->f = NULL; if (ctx->dtl_handle->private_dtl.flux_dtl_handle->msg != NULL) ctx->dtl_handle->private_dtl.flux_dtl_handle->msg = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_finalize (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL) { goto dtl_flux_finalize_done; @@ -247,6 +245,6 @@ dyad_rc_t dyad_dtl_flux_finalize (const dyad_ctx_t* ctx) free (ctx->dtl_handle->private_dtl.flux_dtl_handle); ctx->dtl_handle->private_dtl.flux_dtl_handle = NULL; dtl_flux_finalize_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/ucx_dtl.c b/src/dyad/dtl/ucx_dtl.c index 23b28fb4..f281c8c5 100644 --- a/src/dyad/dtl/ucx_dtl.c +++ b/src/dyad/dtl/ucx_dtl.c @@ -5,17 +5,18 @@ #endif #include -#include -#include -#include -#include +#include #include #include #include #include -#include -#include #include +#include + +#include +#include +#include +#include extern const base64_maps_t base64_maps_rfc4648; @@ -35,11 +36,11 @@ typedef struct ucx_request dyad_ucx_request_t; // initialize our request struct static void dyad_ucx_request_init (void* request) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_ucx_request_t* real_request = NULL; real_request = (dyad_ucx_request_t*)request; real_request->completed = 0; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } #ifndef DYAD_ENABLE_UCX_RMA // Define a function that ucp_tag_msg_recv_nbx will use @@ -50,36 +51,35 @@ static void dyad_recv_callback (void* request, ucs_status_t status, const ucp_tag_recv_info_t* tag_info, void* user_data) -#else // UCP_API_VERSION +#else // UCP_API_VERSION static void dyad_recv_callback (void* request, ucs_status_t status, ucp_tag_recv_info_t* tag_info) -#endif // UCP_API_VERSION +#endif // UCP_API_VERSION { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_ucx_request_t* real_request = NULL; real_request = (dyad_ucx_request_t*)request; real_request->completed = 1; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA #if UCP_API_VERSION >= UCP_VERSION(1, 10) static void dyad_send_callback (void* req, ucs_status_t status, void* ctx) -#else // UCP_API_VERSION +#else // UCP_API_VERSION static void dyad_send_callback (void* req, ucs_status_t status) -#endif // UCP_API_VERSION +#endif // UCP_API_VERSION { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_LOG_STDERR ("Calling send callback"); dyad_ucx_request_t* real_req = (dyad_ucx_request_t*)req; real_req->completed = 1; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } // Simple function used to wait on the async receive -static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, - dyad_ucx_request_t* request) +static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, dyad_ucx_request_t* request) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucs_status_t final_request_status = UCS_OK; if (UCS_OK == request) { DYAD_LOG_INFO (ctx, "Finished Immediately for Request"); @@ -118,15 +118,15 @@ static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, // So, we simply set the status to UCS_OK final_request_status = UCS_OK; dtl_ucx_request_wait_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return final_request_status; } -static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, +static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t* ctx, dyad_dtl_ucx_t* dtl_handle, dyad_dtl_comm_mode_t comm_mode) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_BADBUF; ucs_status_t status; ucp_mem_map_params_t mmap_params; @@ -146,7 +146,7 @@ static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, | UCP_MEM_MAP_PARAM_FIELD_PROT; mmap_params.address = NULL; mmap_params.memory_type = UCS_MEMORY_TYPE_HOST; - mmap_params.length = dtl_handle->max_transfer_size + sizeof(size_t); + mmap_params.length = dtl_handle->max_transfer_size + sizeof (size_t); mmap_params.flags = UCP_MEM_MAP_ALLOCATE; if (dtl_handle->comm_mode == DYAD_COMM_SEND) { mmap_params.prot = UCP_MEM_MAP_PROT_LOCAL_READ; @@ -183,16 +183,16 @@ static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, rc = DYAD_RC_OK; ucx_allocate_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static dyad_rc_t ucx_free_buffer (const dyad_ctx_t *ctx, +static dyad_rc_t ucx_free_buffer (const dyad_ctx_t* ctx, ucp_context_h ucp_ctx, ucp_mem_h mem_handle, void** buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ucp_ctx == NULL) { rc = DYAD_RC_NOCTX; @@ -215,17 +215,20 @@ static dyad_rc_t ucx_free_buffer (const dyad_ctx_t *ctx, rc = DYAD_RC_OK; ucx_free_buf_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_warmup, void* buf, size_t buflen) +static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, + bool is_warmup, + void* buf, + size_t buflen) { /** * The warmup is not used but is passed for consistency with the recv_no_wait calls. */ - (void) is_warmup; - DYAD_C_FUNCTION_START(); + (void)is_warmup; + DYAD_C_FUNCTION_START (); ucs_status_ptr_t stat_ptr = NULL; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; if (dtl_handle->ep == NULL) { @@ -245,18 +248,18 @@ static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_ params.cb.send = dyad_send_callback; DYAD_LOG_INFO (ctx, "Sending data to consumer with ucp_tag_send_nbx"); stat_ptr = ucp_tag_send_nbx (dtl_handle->ep, buf, buflen, dtl_handle->comm_tag, ¶ms); -#else // UCP_API_VERSION - DYAD_LOG_INFO (ctx, "Sending %lu bytes of data to consumer with ucp_tag_send_nb", \ - buflen); +#else // UCP_API_VERSION + DYAD_LOG_INFO (ctx, "Sending %lu bytes of data to consumer with ucp_tag_send_nb", buflen); stat_ptr = ucp_tag_send_nb (dtl_handle->ep, buf, buflen, UCP_DATATYPE_CONTIG, dtl_handle->comm_tag, dyad_send_callback); -#endif // UCP_API_VERSION -#else // DYAD_ENABLE_UCX_RMA - ucs_status_t status = ucp_ep_rkey_unpack (dtl_handle->ep, dtl_handle->rkey_buf, &(dtl_handle->rkey)); +#endif // UCP_API_VERSION +#else // DYAD_ENABLE_UCX_RMA + ucs_status_t status = + ucp_ep_rkey_unpack (dtl_handle->ep, dtl_handle->rkey_buf, &(dtl_handle->rkey)); if (UCX_STATUS_FAIL (status)) { DYAD_LOG_ERROR (ctx, "ucp_ep_rkey_unpack failed"); stat_ptr = (void*)UCS_ERR_NOT_CONNECTED; @@ -265,18 +268,24 @@ static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_ ucp_request_param_t params; params.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK; params.cb.send = dyad_send_callback; - stat_ptr = ucp_put_nbx(dtl_handle->ep, buf, buflen, dtl_handle->cons_buf_ptr, dtl_handle->rkey, ¶ms); - if (UCS_PTR_IS_ERR(stat_ptr)) { - DYAD_LOG_ERROR (ctx, "ucp_put_nbx() failed %s (%d)\n", \ - ucs_status_string(UCS_PTR_STATUS(stat_ptr)), \ - UCS_PTR_STATUS(stat_ptr)); + stat_ptr = ucp_put_nbx (dtl_handle->ep, + buf, + buflen, + dtl_handle->cons_buf_ptr, + dtl_handle->rkey, + ¶ms); + if (UCS_PTR_IS_ERR (stat_ptr)) { + DYAD_LOG_ERROR (ctx, + "ucp_put_nbx() failed %s (%d)\n", + ucs_status_string (UCS_PTR_STATUS (stat_ptr)), + UCS_PTR_STATUS (stat_ptr)); stat_ptr = (void*)UCS_ERR_NOT_CONNECTED; goto ucx_send_no_wait_done; } DYAD_LOG_INFO (ctx, "written data buf of length %lu", buflen); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA ucx_send_no_wait_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return stat_ptr; } @@ -285,7 +294,7 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucs_status_ptr_t stat_ptr = NULL; #ifndef DYAD_ENABLE_UCX_RMA dyad_rc_t rc = DYAD_RC_OK; @@ -355,8 +364,10 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, // The metadata retrived from the probed tag recv event contains // the size of the data to be sent. // So, use that size to allocate a buffer - DYAD_LOG_INFO (ctx, "Got message with tag %lu and size %lu\n", \ - msg_info.sender_tag, msg_info.length); + DYAD_LOG_INFO (ctx, + "Got message with tag %lu and size %lu\n", + msg_info.sender_tag, + msg_info.length); *buflen = msg_info.length; if (is_warmup) { if (*buflen > dtl_handle->max_transfer_size) { @@ -391,36 +402,36 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, recv_params.memory_type = UCS_MEMORY_TYPE_HOST; // Perform the async recv operation using the probed tag recv event stat_ptr = ucp_tag_msg_recv_nbx (dtl_handle->ucx_worker, *buf, *buflen, msg, &recv_params); -#else // UCP_API_VERSION +#else // UCP_API_VERSION stat_ptr = ucp_tag_msg_recv_nb (dtl_handle->ucx_worker, *buf, *buflen, UCP_DATATYPE_CONTIG, msg, dyad_recv_callback); -#endif // UCP_API_VERSION -#else // DYAD_ENABLE_UCX_RMA +#endif // UCP_API_VERSION +#else // DYAD_ENABLE_UCX_RMA dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; ssize_t temp = 0l; do { - memcpy (&temp, dtl_handle->net_buf, sizeof(temp)); - ucp_worker_progress(ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker); - nanosleep((const struct timespec[]){{0, 10000L}}, NULL); + memcpy (&temp, dtl_handle->net_buf, sizeof (temp)); + ucp_worker_progress (ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker); + nanosleep ((const struct timespec[]){{0, 10000L}}, NULL); DYAD_LOG_DEBUG (ctx, "Consumer Waiting for worker to finsih all work"); } while (temp == 0l); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_DEBUG (ctx, "Consumer finsihed all work"); #ifndef DYAD_ENABLE_UCX_RMA ucx_recv_no_wait_done:; -#endif // DYAD_ENABLE_UCX_RMA - DYAD_C_FUNCTION_END(); +#endif // DYAD_ENABLE_UCX_RMA + DYAD_C_FUNCTION_END (); return stat_ptr; } static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; void* send_buf = NULL; void* recv_buf = NULL; @@ -428,7 +439,7 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) #ifndef DYAD_ENABLE_UCX_RMA ucs_status_ptr_t recv_stat_ptr = NULL; size_t recv_buf_len = 0; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA ucs_status_t send_status = UCS_OK; ucs_status_t recv_status = UCS_OK; DYAD_LOG_INFO (ctx, "Starting warmup for UCX DTL"); @@ -468,12 +479,10 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) DYAD_LOG_INFO (ctx, "Starting non-blocking recv for warmup"); recv_stat_ptr = ucx_recv_no_wait (ctx, true, &recv_buf, &recv_buf_len); DYAD_LOG_INFO (ctx, "Waiting on warmup recv to finish"); - recv_status = - dyad_ucx_request_wait (ctx, recv_stat_ptr); -#endif // DYAD_ENABLE_UCX_RMA + recv_status = dyad_ucx_request_wait (ctx, recv_stat_ptr); +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Waiting on warmup send to finish"); - send_status = - dyad_ucx_request_wait (ctx, send_stat_ptr); + send_status = dyad_ucx_request_wait (ctx, send_stat_ptr); DYAD_LOG_INFO (ctx, "Disconnecting from self"); ucx_disconnect (ctx, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker, @@ -487,13 +496,13 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) DYAD_LOG_INFO (ctx, "Communication succeeded (according to UCX)"); #ifndef DYAD_ENABLE_UCX_RMA assert (recv_buf_len == 1); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Correct amount of data received in warmup"); free (recv_buf); rc = DYAD_RC_OK; warmup_region_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -502,7 +511,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, dyad_dtl_comm_mode_t comm_mode, bool debug) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucp_params_t ucx_params; ucp_worker_params_t worker_params; ucp_config_t* config; @@ -513,13 +522,13 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, ctx->dtl_handle->private_dtl.ucx_dtl_handle = malloc (sizeof (struct dyad_dtl_ucx)); if (ctx->dtl_handle->private_dtl.ucx_dtl_handle == NULL) { DYAD_LOG_ERROR (ctx, "Could not allocate UCX DTL context\n"); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_SYSFAIL; } dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; // Allocation/Freeing of the Flux handle should be // handled by the DYAD context - dtl_handle->h = (flux_t*) ctx->h; + dtl_handle->h = (flux_t*)ctx->h; dtl_handle->comm_mode = comm_mode; dtl_handle->debug = debug; dtl_handle->ucx_ctx = NULL; @@ -552,8 +561,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, // * Remote Memory Access communication // * Auto initialization of request objects // * Worker sleep, wakeup, poll, etc. features - ucx_params.field_mask = - UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_REQUEST_SIZE; + ucx_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_REQUEST_SIZE; ucx_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 | UCP_FEATURE_TAG; ucx_params.request_size = sizeof (struct ucx_request); ucx_params.request_init = dyad_ucx_request_init; @@ -585,9 +593,9 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, } // Query the worker for its address DYAD_LOG_INFO (ctx, "Get address of UCP worker\n"); - status = ucp_worker_get_address(dtl_handle->ucx_worker, - &(dtl_handle->local_address), - &(dtl_handle->local_addr_len)); + status = ucp_worker_get_address (dtl_handle->ucx_worker, + &(dtl_handle->local_address), + &(dtl_handle->local_addr_len)); // Initialize endpoint cache rc = dyad_ucx_ep_cache_init (ctx, &(dtl_handle->ep_cache)); @@ -617,7 +625,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, } dtl_handle->ep = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; @@ -625,7 +633,7 @@ error:; // If an error occured, finalize the DTL handle and // return a failing error code dyad_dtl_ucx_finalize (ctx); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_UCXINIT_FAIL; } dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, @@ -633,7 +641,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, uint32_t producer_rank, json_t** restrict packed_obj) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); DYAD_C_FUNCTION_UPDATE_INT ("producer_rank", producer_rank); DYAD_C_FUNCTION_UPDATE_INT ("pid", ctx->pid); @@ -643,7 +651,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, * as we expect a size_t lets put int -1 as a check. */ ssize_t temp = 0; - memcpy (dtl_handle->net_buf, &temp, sizeof(temp)); + memcpy (dtl_handle->net_buf, &temp, sizeof (temp)); dyad_rc_t rc = DYAD_RC_OK; size_t cons_enc_len = 0; char* cons_enc_buf = NULL; @@ -670,8 +678,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, cons_enc_size = base64_encode_using_maps (&base64_maps_rfc4648, cons_enc_buf, cons_enc_len + 1, - (const char*)dtl_handle->local_address, - dtl_handle->local_addr_len); + (const char*)dtl_handle->local_address, + dtl_handle->local_addr_len); if (cons_enc_size < 0) { DYAD_LOG_ERROR (ctx, "Unable to encode address\n"); // TODO log error @@ -699,8 +707,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, rkey_enc_size = base64_encode_using_maps (&base64_maps_rfc4648, rkey_enc_buf, rkey_enc_len + 1, - (const char*)dtl_handle->rkey_buf, - dtl_handle->rkey_size); + (const char*)dtl_handle->rkey_buf, + dtl_handle->rkey_size); if (rkey_enc_size < 0) { DYAD_LOG_ERROR (ctx, "Unable to encode rkey\n"); // TODO log error @@ -717,7 +725,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, // Instead, we use this function to create the tag that will be // used for the upcoming communication. tag_val = 0; - if (flux_get_rank (dtl_handle->h, (uint32_t *)&tag_val) < 0) { + if (flux_get_rank (dtl_handle->h, (uint32_t*)&tag_val) < 0) { DYAD_LOG_ERROR (ctx, "Cannot get consumer rank\n"); rc = DYAD_RC_FLUXFAIL; goto dtl_ucx_rpc_pack_region_finish; @@ -730,13 +738,13 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, // Use Jansson to pack the tag and UCX address into // the payload to be sent via RPC to the producer plugin DYAD_LOG_INFO (ctx, "Packing RPC payload for UCX DTL\n"); -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA char* tag_name = "cons_buf"; uint64_t tag_val = dtl_handle->cons_buf_ptr; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA char tag_val_buf[128]; - memset(tag_val_buf, 0x00, 128); - sprintf( tag_val_buf, "%" PRIu64, tag_val); + memset (tag_val_buf, 0x00, 128); + sprintf (tag_val_buf, "%" PRIu64, tag_val); DYAD_LOG_INFO (ctx, "Creating Json object %lu with buf %s", tag_val, tag_val_buf); *packed_obj = json_pack ("{s:s, s:i, s:s, s:i, s:s%, s:s%}", "upath", @@ -751,8 +759,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, cons_enc_buf, cons_enc_len, "rkey", - rkey_enc_buf, - rkey_enc_len); + rkey_enc_buf, + rkey_enc_len); free (cons_enc_buf); free (rkey_enc_buf); // If the packing failed, log an error @@ -763,13 +771,13 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, } rc = DYAD_RC_OK; dtl_ucx_rpc_pack_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; char* enc_addr = NULL; size_t enc_addr_len = 0; @@ -785,9 +793,9 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, uint64_t tag_val; #ifndef DYAD_ENABLE_UCX_RMA char* tag_name = "tag_cons"; -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA char* tag_name = "cons_buf"; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA char* tag_value_str = NULL; errcode = flux_request_unpack (msg, NULL, @@ -799,7 +807,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, tag_name, &tag_value_str, "pid_cons", - &pid, + &pid, "addr", &enc_addr, &enc_addr_len, @@ -815,9 +823,9 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, } #ifndef DYAD_ENABLE_UCX_RMA tag_cons = tag_val; -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA dtl_handle->cons_buf_ptr = tag_val; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_C_FUNCTION_UPDATE_INT ("pid", pid); DYAD_C_FUNCTION_UPDATE_INT ("tag_cons", tag_cons); dtl_handle->comm_tag = tag_prod << 32 | tag_cons; @@ -870,27 +878,27 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, } rc = DYAD_RC_OK; dtl_ucx_rpc_unpack_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_rpc_respond (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_ucx_rpc_recv_response (const dyad_ctx_t* ctx, flux_future_t* f) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; DYAD_LOG_INFO (dtl_handle, "Validating data_buf in get_buffer"); @@ -907,7 +915,7 @@ dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void // } DYAD_LOG_INFO (dtl_handle, "Validating data_size in get_buffer"); if (data_size > ctx->dtl_handle->private_dtl.ucx_dtl_handle->max_transfer_size) { - DYAD_LOG_ERROR (dtl_handle, \ + DYAD_LOG_ERROR (dtl_handle, "Requested a data size that's larger than the pre-allocated UCX buffer"); rc = DYAD_RC_BADBUF; goto ucx_get_buffer_done; @@ -917,13 +925,13 @@ dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void rc = DYAD_RC_OK; ucx_get_buffer_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_return_buffer (const dyad_ctx_t* ctx, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf == NULL) { rc = DYAD_RC_BADBUF; @@ -931,19 +939,20 @@ dyad_rc_t dyad_dtl_ucx_return_buffer (const dyad_ctx_t* ctx, void** data_buf) } *data_buf = NULL; dtl_ucx_return_buffer_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_establish_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; dyad_dtl_comm_mode_t comm_mode = dtl_handle->comm_mode; if (comm_mode == DYAD_COMM_SEND) { DYAD_LOG_INFO (ctx, "Create UCP endpoint for communication with consumer\n"); - rc = dyad_ucx_ep_cache_find (ctx, dtl_handle->ep_cache, + rc = dyad_ucx_ep_cache_find (ctx, + dtl_handle->ep_cache, dtl_handle->remote_address, dtl_handle->remote_addr_len, &(dtl_handle->ep)); @@ -972,13 +981,13 @@ dyad_rc_t dyad_dtl_ucx_establish_connection (const dyad_ctx_t* ctx) rc = DYAD_RC_BAD_COMM_MODE; } dtl_ucx_establish_connection_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; ucs_status_ptr_t stat_ptr; ucs_status_t status = UCS_OK; @@ -986,24 +995,22 @@ dyad_rc_t dyad_dtl_ucx_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) DYAD_LOG_INFO (ctx, "Processing UCP send request\n"); status = dyad_ucx_request_wait (ctx, stat_ptr); if (status != UCS_OK) { - DYAD_LOG_ERROR (ctx, "UCP Put failed (status = %d)!\n", - (int)status); + DYAD_LOG_ERROR (ctx, "UCP Put failed (status = %d)!\n", (int)status); rc = DYAD_RC_UCXCOMM_FAIL; goto dtl_ucx_send_region_finish; } DYAD_LOG_INFO (ctx, "Data send with UCP succeeded\n"); rc = DYAD_RC_OK; dtl_ucx_send_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; - ucs_status_ptr_t stat_ptr = NULL; // Wait on the recv operation to complete stat_ptr = ucx_recv_no_wait (ctx, false, buf, buflen); @@ -1018,9 +1025,9 @@ dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) rc = DYAD_RC_UCXCOMM_FAIL; goto dtl_ucx_recv_region_finish; } -#else // DYAD_ENABLE_UCX_RMA - (void)stat_ptr; -#endif // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA + (void)stat_ptr; +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Data receive using UCX is successful\n"); DYAD_LOG_INFO (ctx, "Received %lu bytes from producer\n", *buflen); @@ -1028,14 +1035,14 @@ dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) rc = DYAD_RC_OK; #ifndef DYAD_ENABLE_UCX_RMA dtl_ucx_recv_region_finish:; -#endif // DYAD_ENABLE_UCX_RMA - DYAD_C_FUNCTION_END(); +#endif // DYAD_ENABLE_UCX_RMA + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; dyad_dtl_comm_mode_t comm_mode = dtl_handle->comm_mode; @@ -1049,8 +1056,8 @@ dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) // "released."); // } #ifdef DYAD_ENABLE_UCX_RMA - ucp_rkey_destroy(dtl_handle->rkey); -#endif // DYAD_ENABLE_UCX_RMA + ucp_rkey_destroy (dtl_handle->rkey); +#endif // DYAD_ENABLE_UCX_RMA dtl_handle->ep = NULL; // Sender doesn't have a consumer address at this time // So, free the consumer address when closing the connection @@ -1078,13 +1085,13 @@ dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) // TODO create new RC for this case rc = DYAD_RC_BAD_COMM_MODE; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_dtl_ucx_t* dtl_handle = NULL; dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL || ctx->dtl_handle->private_dtl.ucx_dtl_handle == NULL) { @@ -1108,10 +1115,7 @@ dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) } // Free memory buffer if not already freed if (dtl_handle->mem_handle != NULL) { - ucx_free_buffer (ctx, - dtl_handle->ucx_ctx, - dtl_handle->mem_handle, - &(dtl_handle->net_buf)); + ucx_free_buffer (ctx, dtl_handle->ucx_ctx, dtl_handle->mem_handle, &(dtl_handle->net_buf)); } // Release worker if not already released if (dtl_handle->ucx_worker != NULL) { @@ -1131,6 +1135,6 @@ dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) ctx->dtl_handle->private_dtl.ucx_dtl_handle = NULL; rc = DYAD_RC_OK; dtl_ucx_finalize_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/ucx_dtl.h b/src/dyad/dtl/ucx_dtl.h index f8af6e8f..1572dc9c 100644 --- a/src/dyad/dtl/ucx_dtl.h +++ b/src/dyad/dtl/ucx_dtl.h @@ -7,10 +7,11 @@ #error "no config" #endif +#include +#include + #include #include -#include -#include struct dyad_dtl_ucx { flux_t* h; @@ -34,7 +35,7 @@ struct dyad_dtl_ucx { size_t rkey_size; uint64_t cons_buf_ptr; // Internal for Sender - ucp_rkey_h rkey; + ucp_rkey_h rkey; }; typedef struct dyad_dtl_ucx dyad_dtl_ucx_t; @@ -45,9 +46,9 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, bool debug); dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, - const char* upath, + const char* upath, uint32_t producer_rank, - json_t** packed_obj); + json_t** packed_obj); dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath); diff --git a/src/dyad/dtl/ucx_ep_cache.cpp b/src/dyad/dtl/ucx_ep_cache.cpp index 7d33c619..aa9501ea 100644 --- a/src/dyad/dtl/ucx_ep_cache.cpp +++ b/src/dyad/dtl/ucx_ep_cache.cpp @@ -4,34 +4,34 @@ #error "no config" #endif -#include -#include -#include -#include - #include #include #include #include +#include +#include +#include +#include + using key_type = uint64_t; using cache_type = std::unordered_map; -static void __attribute__((unused)) +static void __attribute__ ((unused)) dyad_ucx_ep_err_handler (void* arg, ucp_ep_h ep, ucs_status_t status) { - DYAD_C_FUNCTION_START(); - dyad_ctx_t __attribute__((unused)) *ctx = (dyad_ctx_t*)arg; + DYAD_C_FUNCTION_START (); + dyad_ctx_t __attribute__ ((unused))* ctx = (dyad_ctx_t*)arg; DYAD_LOG_ERROR (ctx, "An error occured on the UCP endpoint (status = %d)", status); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } -dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, +dyad_rc_t ucx_connect (const dyad_ctx_t* ctx, ucp_worker_h worker, const ucp_address_t* addr, ucp_ep_h* ep) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; ucp_ep_params_t params; ucs_status_t status = UCS_OK; @@ -49,13 +49,13 @@ dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, goto ucx_connect_done; } ucx_connect_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep) +dyad_rc_t ucx_disconnect (const dyad_ctx_t* ctx, ucp_worker_h worker, ucp_ep_h ep) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; ucs_status_t status = UCS_OK; ucs_status_ptr_t stat_ptr; @@ -100,13 +100,13 @@ dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h e } } ucx_disconnect_region_finish: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache) +dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t* ctx, ucx_ep_cache_h* cache) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (cache == nullptr || *cache != nullptr) { rc = DYAD_RC_BADBUF; @@ -118,17 +118,17 @@ dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache) goto ucx_ep_cache_init_done; } ucx_ep_cache_init_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t* ctx, const ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_ep_h* ep) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ep == nullptr || *ep != nullptr) { rc = DYAD_RC_BADBUF; @@ -150,22 +150,23 @@ dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, rc = DYAD_RC_SYSFAIL; } ucx_ep_cache_find_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t* ctx, ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; try { cache_type* cpp_cache = reinterpret_cast (cache); uint64_t key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - DYAD_C_FUNCTION_UPDATE_INT("cons_key", ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) + DYAD_C_FUNCTION_UPDATE_INT ("cons_key", + ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) auto cache_it = cpp_cache->find (key); if (cache_it != cpp_cache->end ()) { rc = DYAD_RC_OK; @@ -173,43 +174,43 @@ dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, DYAD_LOG_INFO (ctx, "No cache entry found. Creating new connection"); rc = ucx_connect (ctx, worker, addr, &ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); if (!DYAD_IS_ERROR (rc)) { - cpp_cache->insert_or_assign(key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); + cpp_cache->insert_or_assign (key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); rc = DYAD_RC_OK; } } } catch (...) { rc = DYAD_RC_SYSFAIL; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static inline cache_type::iterator cache_remove_impl (const dyad_ctx_t *ctx, +static inline cache_type::iterator cache_remove_impl (const dyad_ctx_t* ctx, cache_type* cache, cache_type::iterator it, ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); if (it != cache->end ()) { ucx_disconnect (ctx, worker, it->second); // The UCP address was allocated with 'malloc' while unpacking // the RPC message. So, we extract it from the key and free // it after erasing the iterator auto next_it = cache->erase (it); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return next_it; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return cache->end (); } -dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t* ctx, ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; try { cache_type* cpp_cache = reinterpret_cast (cache); @@ -220,13 +221,15 @@ dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, } catch (...) { rc = DYAD_RC_SYSFAIL; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache, ucp_worker_h worker) +dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t* ctx, + ucx_ep_cache_h* cache, + ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); if (cache == nullptr || *cache == nullptr) { return DYAD_RC_OK; } @@ -236,6 +239,6 @@ dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, ucx_ep_cache_h* cac } delete cpp_cache; *cache = nullptr; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } diff --git a/src/dyad/dtl/ucx_ep_cache.h b/src/dyad/dtl/ucx_ep_cache.h index aaa88c54..55d079d9 100644 --- a/src/dyad/dtl/ucx_ep_cache.h +++ b/src/dyad/dtl/ucx_ep_cache.h @@ -8,10 +8,11 @@ #endif #include +#include + #include #include #include -#include #ifdef __cplusplus extern "C" { @@ -21,38 +22,36 @@ extern "C" { // of UCX operations #define UCX_STATUS_FAIL(status) (status != UCS_OK) - - -dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, +dyad_rc_t ucx_connect (const dyad_ctx_t* ctx, ucp_worker_h worker, const ucp_address_t* addr, ucp_ep_h* ep); -dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep); +dyad_rc_t ucx_disconnect (const dyad_ctx_t* ctx, ucp_worker_h worker, ucp_ep_h ep); // NOTE: in future, add option to configure replacement strategy -dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache); +dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t* ctx, ucx_ep_cache_h* cache); // NOTE: not positive if UCP addresses are 100% unique by worker -dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t* ctx, const ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_ep_h* ep); -dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t* ctx, ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_worker_h worker); -dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t* ctx, ucx_ep_cache_h cache, const ucp_address_t* addr, const size_t addr_size, ucp_worker_h worker); -dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, +dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t* ctx, ucx_ep_cache_h* cache, ucp_worker_h worker); diff --git a/src/dyad/modules/dyad.c b/src/dyad/modules/dyad.c index 31dce08a..c22f3d87 100644 --- a/src/dyad/modules/dyad.c +++ b/src/dyad/modules/dyad.c @@ -14,18 +14,6 @@ #error "no config" #endif -// #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #if defined(__cplusplus) #include #include @@ -49,6 +37,18 @@ #include #include +// #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #define TIME_DIFF(Tstart, Tend) \ ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - (Tstart).tv_nsec) \ / 1000000000L) @@ -197,7 +197,11 @@ dyad_fetch_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, inlen = read (fd, inbuf, file_size); #endif if (inlen != file_size) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", fullpath, inlen, file_size); + DYAD_LOG_ERROR (mod_ctx->ctx, + "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", + fullpath, + inlen, + file_size); goto fetch_error; } #ifdef DYAD_ENABLE_UCX_RMA @@ -230,7 +234,9 @@ dyad_fetch_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, } DYAD_LOG_DEBUG (mod_ctx->ctx, "Close RPC message stream with an ENODATA (%d) message", ENODATA); if (flux_respond_error (h, msg, ENODATA, NULL) < 0) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", __func__); + DYAD_LOG_ERROR (mod_ctx->ctx, + "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", + __func__); } DYAD_LOG_INFO (mod_ctx->ctx, "Finished %s module invocation\n", DYAD_DTL_RPC_NAME); goto end_fetch_cb; @@ -351,8 +357,10 @@ static int opt_parse (opt_parse_out_t *restrict opt, // mode as the option, then skip reinitializing DYAD_LOG_STDERR ("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); opt->dtl_mode = optarg; - if (strcmp("UCX", optarg) == 0) *dtl_mode = DYAD_DTL_UCX; - else if (strcmp("FLUX_RPC", optarg) == 0) *dtl_mode = DYAD_DTL_FLUX_RPC; + if (strcmp ("UCX", optarg) == 0) + *dtl_mode = DYAD_DTL_UCX; + else if (strcmp ("FLUX_RPC", optarg) == 0) + *dtl_mode = DYAD_DTL_FLUX_RPC; break; case 'i': #ifndef DYAD_LOGGER_NO_LOG @@ -423,12 +431,12 @@ dyad_rc_t dyad_module_ctx_init (const opt_parse_out_t *opt, flux_t *h) DYAD_DTL_MODE_ENV, getenv (DYAD_DTL_MODE_ENV)); } - char* kvs = getenv("DYAD_KVS_NAMESPACE"); + char *kvs = getenv ("DYAD_KVS_NAMESPACE"); if (kvs != NULL) { - DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is set to `%s'\n", kvs); + DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is set to `%s'\n", kvs); } else { DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is not set\n"); - } + } dyad_ctx_init (DYAD_COMM_SEND, h); mod_ctx->ctx = dyad_ctx_get (); dyad_ctx_t *ctx = mod_ctx->ctx; diff --git a/src/dyad/modules/test_opt_parse.c b/src/dyad/modules/test_opt_parse.c index 04293e87..6c9ecfa0 100644 --- a/src/dyad/modules/test_opt_parse.c +++ b/src/dyad/modules/test_opt_parse.c @@ -7,11 +7,6 @@ #define DYAD_UTIL_LOGGER 1 #define DYAD_LOGGER_FLUX 1 -#include -#include -#include -#include -#include #include #if defined(__cplusplus) @@ -32,10 +27,16 @@ #endif // defined(__cplusplus) #include +#include #include #include #include -#include + +#include +#include +#include +#include +#include /** This is a temporary measure until environment variable based initialization * is implemented */ @@ -54,13 +55,16 @@ static dyad_dtl_mode_t get_dtl_mode_env () DYAD_LOG_STDERR ("DYAD MOD: UCX DTL mode found in environment\n"); return DYAD_DTL_UCX; } else { - DYAD_LOG_STDERR ("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", \ - DYAD_DTL_MODE_ENV, e, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); + DYAD_LOG_STDERR ("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, + e, + dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); return DYAD_DTL_DEFAULT; } } else { - DYAD_LOG_STDERR ("DYAD MOD: %s is not set. Defaulting to %s\n", \ - DYAD_DTL_MODE_ENV, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); + DYAD_LOG_STDERR ("DYAD MOD: %s is not set. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, + dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); return DYAD_DTL_DEFAULT; } return DYAD_DTL_DEFAULT; @@ -71,44 +75,46 @@ static void show_help (void) DYAD_LOG_STDOUT ("dyad module options and arguments\n"); DYAD_LOG_STDOUT (" -h, --help: Show help.\n"); DYAD_LOG_STDOUT (" -d, --debug: Enable debugging log message.\n"); - DYAD_LOG_STDOUT (" -m, --mode: DTL mode. Need an argument.\n" - " Either 'FLUX_RPC' (default) or 'UCX'.\n"); - DYAD_LOG_STDOUT (" -i, --info_log: Specify the file into which to redirect\n" - " info logging. Does nothing if DYAD was not\n" - " configured with '-DDYAD_LOGGER=PRINTF'.\n" - " Need a filename as an argument.\n"); - DYAD_LOG_STDOUT (" -e, --error_log: Specify the file into which to redirect\n" - " error logging. Does nothing if DYAD was\n" - " not configured with '-DDYAD_LOGGER=PRINTF'\n" - " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT ( + " -m, --mode: DTL mode. Need an argument.\n" + " Either 'FLUX_RPC' (default) or 'UCX'.\n"); + DYAD_LOG_STDOUT ( + " -i, --info_log: Specify the file into which to redirect\n" + " info logging. Does nothing if DYAD was not\n" + " configured with '-DDYAD_LOGGER=PRINTF'.\n" + " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT ( + " -e, --error_log: Specify the file into which to redirect\n" + " error logging. Does nothing if DYAD was\n" + " not configured with '-DDYAD_LOGGER=PRINTF'\n" + " Need a filename as an argument.\n"); } -static int opt_parse (dyad_ctx_t* ctx, const unsigned broker_rank, - dyad_dtl_mode_t *dtl_mode, int argc, char** argv) +static int opt_parse (dyad_ctx_t* ctx, + const unsigned broker_rank, + dyad_dtl_mode_t* dtl_mode, + int argc, + char** argv) { size_t arglen = 0ul; - #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[PATH_MAX+1] = {'\0'}; - char err_file_name[PATH_MAX+1] = {'\0'}; +#ifndef DYAD_LOGGER_NO_LOG + char log_file_name[PATH_MAX + 1] = {'\0'}; + char err_file_name[PATH_MAX + 1] = {'\0'}; sprintf (log_file_name, "dyad_mod_%u.out", broker_rank); sprintf (err_file_name, "dyad_mod_%d.err", broker_rank); - #endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG *dtl_mode = DYAD_DTL_END; char* prod_managed_path = NULL; bool debug = false; - (void) debug; - - while (1) - { - static struct option long_options[] = - { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"mode", required_argument, 0, 'm'}, - {"info_log", required_argument, 0, 'i'}, - {"error_log", required_argument, 0, 'e'}, - {0, 0, 0, 0} - }; + (void)debug; + + while (1) { + static struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"debug", no_argument, 0, 'd'}, + {"mode", required_argument, 0, 'm'}, + {"info_log", required_argument, 0, 'i'}, + {"error_log", required_argument, 0, 'e'}, + {0, 0, 0, 0}}; /* getopt_long stores the option index here. */ int option_index = 0; int c = -1; @@ -120,10 +126,9 @@ static int opt_parse (dyad_ctx_t* ctx, const unsigned broker_rank, DYAD_LOG_DEBUG (ctx, "DYAD_MOD: no more option\n"); break; } - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: opt %c, index %d\n", (char) c, optind); + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: opt %c, index %d\n", (char)c, optind); - switch (c) - { + switch (c) { case 'h': show_help (); break; @@ -142,34 +147,34 @@ static int opt_parse (dyad_ctx_t* ctx, const unsigned broker_rank, DYAD_LOG_ERROR (ctx, "DYAD_MOD: Invalid DTL 'mode' (%s) provided\n", optarg); *dtl_mode = DYAD_DTL_END; show_help (); - return DYAD_RC_SYSFAIL ; + return DYAD_RC_SYSFAIL; } break; case 'i': - #ifndef DYAD_LOGGER_NO_LOG +#ifndef DYAD_LOGGER_NO_LOG DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'info_log' option -i with value `%s'\n", optarg); sprintf (log_file_name, "%s_%u.out", optarg, broker_rank); - #endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG break; case 'e': - #ifndef DYAD_LOGGER_NO_LOG +#ifndef DYAD_LOGGER_NO_LOG DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'error_log' option -e with value `%s'\n", optarg); sprintf (err_file_name, "%s_%d.err", optarg, broker_rank); - #endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG break; case '?': /* getopt_long already printed an error message. */ break; default: DYAD_LOG_DEBUG (ctx, "DYAD_MOD: option parsing failed %d\n", c); - return DYAD_RC_SYSFAIL ; + return DYAD_RC_SYSFAIL; } } - #ifndef DYAD_LOGGER_NO_LOG +#ifndef DYAD_LOGGER_NO_LOG DYAD_LOG_STDOUT_REDIRECT (log_file_name); DYAD_LOG_STDERR_REDIRECT (err_file_name); - #endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG if (*dtl_mode == DYAD_DTL_END) { DYAD_LOG_DEBUG (ctx, "DYAD_MOD: Did not find DTL 'mode' option"); @@ -187,20 +192,23 @@ static int opt_parse (dyad_ctx_t* ctx, const unsigned broker_rank, const char* tmp_ptr = prod_managed_path; prod_managed_path = (char*)malloc (prod_path_len + 1); if (prod_managed_path == NULL) { - DYAD_LOG_ERROR (ctx, "DYAD_MOD: Could not allocate buffer for " \ - "Producer managed path!\n"); - return DYAD_RC_SYSFAIL ; + DYAD_LOG_ERROR (ctx, + "DYAD_MOD: Could not allocate buffer for " + "Producer managed path!\n"); + return DYAD_RC_SYSFAIL; } strncpy (prod_managed_path, tmp_ptr, prod_path_len + 1); - DYAD_LOG_INFO (ctx, "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", \ - prod_managed_path, dyad_dtl_mode_name[*dtl_mode]); + DYAD_LOG_INFO (ctx, + "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", + prod_managed_path, + dyad_dtl_mode_name[*dtl_mode]); } - DYAD_LOG_INFO (NULL, "DYAD_MOD: debug flag set to %s\n", debug? "true": "false"); + DYAD_LOG_INFO (NULL, "DYAD_MOD: debug flag set to %s\n", debug ? "true" : "false"); DYAD_LOG_INFO (NULL, "DYAD_MOD: prod_managed_path set to %s\n", prod_managed_path); return DYAD_RC_OK; } -int main (int argc, char **argv) +int main (int argc, char** argv) { dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; uint32_t broker_rank = 0u; @@ -213,4 +221,3 @@ int main (int argc, char **argv) return EXIT_SUCCESS; } - diff --git a/src/dyad/stream/dyad_stream_core.cpp b/src/dyad/stream/dyad_stream_core.cpp index 900461bf..a4b87e26 100644 --- a/src/dyad/stream/dyad_stream_core.cpp +++ b/src/dyad/stream/dyad_stream_core.cpp @@ -14,18 +14,14 @@ #error "no config" #endif -#include - -#include -#include -#include -#include - #ifndef _GNU_SOURCE #define _GNU_SOURCE #include #endif // _GNU_SOURCE +#include +#include // dirname +#include #include #include #include @@ -33,17 +29,22 @@ #include #include #include + using namespace std; // std::clock () // #include // c++11 #include + +#include + #include #include #include #include -#include -#include // dirname -#include +#include +#include +#include +#include namespace dyad { @@ -54,9 +55,13 @@ namespace dyad *****************************************************************************/ dyad_stream_core::dyad_stream_core () - : m_ctx (NULL), m_ctx_mutable (NULL), m_initialized (false), m_is_prod (false), m_is_cons (false) + : m_ctx (NULL), + m_ctx_mutable (NULL), + m_initialized (false), + m_is_prod (false), + m_is_cons (false) { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); } dyad_stream_core::~dyad_stream_core () @@ -66,9 +71,9 @@ dyad_stream_core::~dyad_stream_core () void dyad_stream_core::finalize () { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); if (m_ctx != NULL) { - //dyad_finalize (&m_ctx); + // dyad_finalize (&m_ctx); m_ctx = m_ctx_mutable = NULL; m_initialized = false; } @@ -76,7 +81,7 @@ void dyad_stream_core::finalize () void dyad_stream_core::init (const bool reinit) { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); bool reinit_env = false; char *e = NULL; @@ -101,9 +106,7 @@ void dyad_stream_core::init (const bool reinit) m_is_prod = false; } - if (reinit || reinit_env || - !(m_ctx = m_ctx_mutable = dyad_ctx_get ())) - { + if (reinit || reinit_env || !(m_ctx = m_ctx_mutable = dyad_ctx_get ())) { dyad_ctx_init (DYAD_COMM_RECV, NULL); m_ctx = m_ctx_mutable = dyad_ctx_get (); log_info ("Stream core is initialized by env variables."); @@ -117,8 +120,8 @@ void dyad_stream_core::init (const bool reinit) void dyad_stream_core::init (const dyad_params &p) { - DYAD_CPP_FUNCTION(); - DYAD_LOG_DEBUG(m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); + DYAD_CPP_FUNCTION (); + DYAD_LOG_DEBUG (m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); dyad_rc_t rc = dyad_init (p.m_debug, false, p.m_shared_storage, @@ -133,14 +136,15 @@ void dyad_stream_core::init (const dyad_params &p) p.m_cons_managed_path.c_str (), p.m_relative_to_managed_path, dyad_dtl_mode_name[static_cast (p.m_dtl_mode)], - DYAD_COMM_RECV, NULL); + DYAD_COMM_RECV, + NULL); #if defined(DYAD_HAS_STD_FSTREAM_FD) m_ctx_mutable->use_fs_locks = true; #else // Rely on the KVS-based synchronization and disable checking for fs lock based logic. m_ctx_mutable->use_fs_locks = false; #endif - (void) rc; + (void)rc; // TODO figure out if we want to error if init fails m_initialized = true; log_info ("Stream core is initialized by parameters"); @@ -148,20 +152,24 @@ void dyad_stream_core::init (const dyad_params &p) void dyad_stream_core::log_info (const std::string &msg_head) const { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); DYAD_LOG_INFO (m_ctx, "=== %s ===", msg_head.c_str ()); DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_CONSUMER_ENV, m_ctx->cons_managed_path); DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_PRODUCER_ENV, m_ctx->prod_managed_path); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_RELATIVE_ENV, + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_PATH_RELATIVE_ENV, (m_ctx->relative_to_managed_path) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, - (m_ctx->debug) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SHARED_STORAGE_ENV, + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, (m_ctx->debug) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_SHARED_STORAGE_ENV, (m_ctx->shared_storage) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_ASYNC_PUBLISH_ENV, + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_ASYNC_PUBLISH_ENV, (m_ctx->async_publish) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, - (m_ctx->fsync_write) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, (m_ctx->fsync_write) ? "true" : "false"); DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_BINS_ENV, m_ctx->key_bins); DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_SERVICE_MUX_ENV, m_ctx->service_mux); @@ -181,7 +189,7 @@ bool dyad_stream_core::is_dyad_consumer () const bool dyad_stream_core::open_sync (const char *path) { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); DYAD_CPP_FUNCTION_UPDATE ("path", path); DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC OPEN: enters sync (\"%s\").", path); if (!m_initialized) { @@ -200,13 +208,13 @@ bool dyad_stream_core::open_sync (const char *path) return false; } - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); + DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); return true; } bool dyad_stream_core::close_sync (const char *path) { - DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION (); DYAD_CPP_FUNCTION_UPDATE ("path", path); DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC CLOSE: enters sync (\"%s\").\n", path); if (!m_initialized) { @@ -277,8 +285,7 @@ int dyad_stream_core::file_unlock (int fd) const return dyad_release_flock (m_ctx, fd, &a_flock); } -bool dyad_stream_core::cmp_canonical_path_prefix ( - bool is_prod, const char* const __restrict__ path) +bool dyad_stream_core::cmp_canonical_path_prefix (bool is_prod, const char *const __restrict__ path) { memset (upath, '\0', PATH_MAX); return ::cmp_canonical_path_prefix (m_ctx, is_prod, path, upath, PATH_MAX); @@ -286,7 +293,7 @@ bool dyad_stream_core::cmp_canonical_path_prefix ( std::string dyad_stream_core::get_upath () const { - return std::string {upath}; + return std::string{upath}; } } // end of namespace dyad diff --git a/src/dyad/utils/base64/base64.c b/src/dyad/utils/base64/base64.c index 5fa7f114..2599287d 100644 --- a/src/dyad/utils/base64/base64.c +++ b/src/dyad/utils/base64/base64.c @@ -59,9 +59,7 @@ size_t base64_encoded_length (size_t srclen) return ((srclen + 2) / 3) * 4; } -void base64_encode_triplet_using_maps (const base64_maps_t *maps, - char dest[4], - const char src[3]) +void base64_encode_triplet_using_maps (const base64_maps_t *maps, char dest[4], const char src[3]) { char a = src[0]; char b = src[1]; diff --git a/src/dyad/utils/base64/base64.h b/src/dyad/utils/base64/base64.h index 898f3d2b..fb4011b0 100644 --- a/src/dyad/utils/base64/base64.h +++ b/src/dyad/utils/base64/base64.h @@ -53,9 +53,7 @@ void base64_init_maps (base64_maps_t *dest, const char src[64]); * @param dest Buffer containing 3 bytes * @param src Buffer containing 4 characters */ -void base64_encode_triplet_using_maps (const base64_maps_t *maps, - char dest[4], - const char src[3]); +void base64_encode_triplet_using_maps (const base64_maps_t *maps, char dest[4], const char src[3]); /** * base64_encode_tail_using_maps - encode the final bytes of a source using a specific @@ -164,10 +162,7 @@ extern const base64_maps_t base64_maps_rfc4648; * encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src)); * printf("Returned data of length %zd @%p", encoded_length, &dest); */ -static inline ssize_t base64_encode (char *dest, - size_t destlen, - const char *src, - size_t srclen) +static inline ssize_t base64_encode (char *dest, size_t destlen, const char *src, size_t srclen) { return base64_encode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); } @@ -214,10 +209,7 @@ static inline void base64_encode_tail (char dest[4], const char *src, size_t src * decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src)); * printf("Returned data of length %zd @%p", decoded_length, &ret); */ -static inline ssize_t base64_decode (char *dest, - size_t destlen, - const char *src, - size_t srclen) +static inline ssize_t base64_decode (char *dest, size_t destlen, const char *src, size_t srclen) { return base64_decode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); } diff --git a/src/dyad/utils/test_cmp_canonical_path_prefix.c b/src/dyad/utils/test_cmp_canonical_path_prefix.c index 7a92e9ec..d92fcd79 100644 --- a/src/dyad/utils/test_cmp_canonical_path_prefix.c +++ b/src/dyad/utils/test_cmp_canonical_path_prefix.c @@ -4,13 +4,14 @@ #error "no config" #endif -#include -#include -#include #include #include +#include #include +#include +#include + int main (int argc, char** argv) { char* prefix = NULL; @@ -32,7 +33,8 @@ int main (int argc, char** argv) } const size_t prefix_len = strlen (prefix); - const size_t can_prefix_len = can_prefix? strlen (can_prefix) : 0u;; + const size_t can_prefix_len = can_prefix ? strlen (can_prefix) : 0u; + ; uint32_t prefix_hash = hash_str (prefix, DYAD_SEED); uint32_t can_prefix_hash = hash_str (can_prefix, DYAD_SEED); diff --git a/src/dyad/utils/test_murmur3.c b/src/dyad/utils/test_murmur3.c index 9bb64f78..6c9627fd 100644 --- a/src/dyad/utils/test_murmur3.c +++ b/src/dyad/utils/test_murmur3.c @@ -1,8 +1,9 @@ -#include "dyad/utils/murmur3.h" +#include #include -#include #include -#include +#include + +#include "dyad/utils/murmur3.h" static int gen_path_key (const char* restrict str, char* restrict path_key, @@ -42,7 +43,7 @@ static int gen_path_key (const char* restrict str, MurmurHash3_x64_128 (str_long, str_len, seed, hash); uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; n = snprintf (path_key + cx, len - cx, "%x.", bin); - //n = snprintf (path_key + cx, len - cx, "%x%x%x%x.", hash[0], hash[1], hash[2], hash[3]); + // n = snprintf (path_key + cx, len - cx, "%x%x%x%x.", hash[0], hash[1], hash[2], hash[3]); cx += n; if (cx >= len || n < 0) { return -1; @@ -56,7 +57,6 @@ static int gen_path_key (const char* restrict str, return 0; } - int main (int argc, char** argv) { if (argc < 4) { @@ -67,9 +67,9 @@ int main (int argc, char** argv) int depth = atoi (argv[1]); int width = atoi (argv[2]); for (int i = 3; i < argc; i++) { - char path_key [PATH_MAX + 1] = {'\0'}; + char path_key[PATH_MAX + 1] = {'\0'}; gen_path_key (argv[i], path_key, PATH_MAX, depth, width); - printf("%s\t%s\n", argv[i], path_key); + printf ("%s\t%s\n", argv[i], path_key); } return EXIT_SUCCESS; diff --git a/src/dyad/utils/utils.c b/src/dyad/utils/utils.c index 03184052..e8e00ca2 100644 --- a/src/dyad/utils/utils.c +++ b/src/dyad/utils/utils.c @@ -21,41 +21,41 @@ // logger for utils where it does not depend on flux #define DYAD_UTIL_LOGGER 1 -#include - -#include -#include -#include - -#include // open -#include // basename dirname +#include // open +#include // basename dirname +#include #include // open #include // open -#include #include // readlink #if defined(__cplusplus) #include #include #include +#include #include #include #include #include -#include // #include // c++11 #else #include #include // PATH_MAX #include #include +#include #include #include #include #include -#include #endif +#include + +#include +#include +#include + #ifndef DYAD_PATH_DELIM #define DYAD_PATH_DELIM "/" #endif @@ -63,10 +63,12 @@ /// If hashing is not possible, returns 0. Otherwise, returns a non-zero hash value. uint32_t hash_str (const char* str, const uint32_t seed) { - if (!str) return 0u; + if (!str) + return 0u; const char* str_long = str; size_t str_len = strlen (str); - if (str_len == 0ul) return 0u; + if (str_len == 0ul) + return 0u; // Just append the string so that it can be as large as 128 bytes. if (str_len < 128ul) { @@ -85,8 +87,7 @@ uint32_t hash_str (const char* str, const uint32_t seed) /** If hashing is not possible, returns 0. Otherwise, returns a non-zero hash value. * This only hashes the prefix of a given length */ -uint32_t hash_path_prefix (const char* str, const uint32_t seed, - const size_t len) +uint32_t hash_path_prefix (const char* str, const uint32_t seed, const size_t len) { if (!str || len == 0ul) { return 0u; @@ -94,7 +95,8 @@ uint32_t hash_path_prefix (const char* str, const uint32_t seed, const char* str_long = str; size_t str_len = strlen (str); - if (str_len < len) return 0u; + if (str_len < len) + return 0u; str_len = len; // Just append the string so that it can be as large as 128 bytes. @@ -139,9 +141,8 @@ char* concat_str (char* __restrict__ str, const size_t str_len = (con_end ? (str_len_org - con_len) : str_len_org); const char* const str_end = str + str_capacity; - bool no_overlap = - ((to_append + strlen (to_append) <= str) || (str_end <= to_append)) - && ((connector + strlen (connector) <= str) || (str_end <= connector)); + bool no_overlap = ((to_append + strlen (to_append) <= str) || (str_end <= to_append)) + && ((connector + strlen (connector) <= str) || (str_end <= connector)); if (!no_overlap) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: buffers overlap.\n"); @@ -188,10 +189,9 @@ bool extract_user_path (const char* __restrict__ prefix, { const char* const upath_end = upath + upath_capacity; - bool no_overlap = - ((prefix + prefix_len <= upath) || (upath_end <= prefix)) - && ((full + full_len <= upath) || (upath_end <= full)) - && ((delim + delim_len <= upath) || (upath_end <= delim)); + bool no_overlap = ((prefix + prefix_len <= upath) || (upath_end <= prefix)) + && ((full + full_len <= upath) || (upath_end <= full)) + && ((delim + delim_len <= upath) || (upath_end <= delim)); if (!no_overlap) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: buffers overlap.\n"); @@ -257,7 +257,7 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, const size_t upath_capacity) { // Only works when there are no multiple absolute paths via hardlinks - char can_path[PATH_MAX] = {'\0'}; // canonical form of the given path + char can_path[PATH_MAX] = {'\0'}; // canonical form of the given path if (!ctx) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: Invalid dyad context!\n"); return false; @@ -285,11 +285,10 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, can_prefix_hash = ctx->cons_real_hash; } - const uint32_t path_hash1 = hash_path_prefix (path, DYAD_SEED, prefix_len); - if ((path_hash1 == prefix_hash) && - extract_user_path (prefix, path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((path_hash1 == prefix_hash) + && extract_user_path (prefix, path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } @@ -311,15 +310,15 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, const uint32_t can_path_hash1 = hash_path_prefix (can_path, DYAD_SEED, prefix_len); - if ((can_path_hash1 == prefix_hash) && - extract_user_path (prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((can_path_hash1 == prefix_hash) + && extract_user_path (prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } if (can_prefix_len > 0u) { const uint32_t can_path_hash2 = hash_path_prefix (can_path, DYAD_SEED, can_prefix_len); - if ((can_path_hash2 == can_prefix_hash) && - extract_user_path (can_prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((can_path_hash2 == can_prefix_hash) + && extract_user_path (can_prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } } @@ -365,12 +364,12 @@ int mkdir_as_needed (const char* path, const mode_t m) return -2; } else if ((sb.st_mode & RWX_UGO) ^ (m & RWX_UGO)) { DYAD_LOG_DEBUG (NULL, - "Directory \"%s\" already exists with " - "different permission bits %o from " - "the requested %o\n", - path, - (sb.st_mode & RWX_UGO), - (m & RWX_UGO)); + "Directory \"%s\" already exists with " + "different permission bits %o from " + "the requested %o\n", + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists @@ -381,16 +380,16 @@ int mkdir_as_needed (const char* path, const mode_t m) if (mkpath (path, m) != 0) { if (stat (path, &sb) == 0) { // already exist if (S_ISDIR (sb.st_mode) == 0) { // not a directory - DYAD_LOG_DEBUG (NULL,"\"%s\" already exists not as a directory\n", path); + DYAD_LOG_DEBUG (NULL, "\"%s\" already exists not as a directory\n", path); return -4; } else if ((sb.st_mode & RWX_UGO) ^ (m & RWX_UGO)) { DYAD_LOG_DEBUG (NULL, - "Directory \"%s\" already exists with " - "different permission bits %o from " - "the requested %o\n", - path, - (sb.st_mode & RWX_UGO), - (m & RWX_UGO)); + "Directory \"%s\" already exists with " + "different permission bits %o from " + "the requested %o\n", + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists @@ -420,9 +419,10 @@ int get_path (const int fd, const size_t max_size, char* path) sprintf (proclink, "/proc/self/fd/%d", fd); ssize_t rc = readlink (proclink, path, max_size); if (rc < (ssize_t)0) { - DYAD_LOG_DEBUG (NULL, "DYAD UTIL: error reading the file link (%s): %s\n", - strerror (errno), - proclink); + DYAD_LOG_DEBUG (NULL, + "DYAD UTIL: error reading the file link (%s): %s\n", + strerror (errno), + proclink); return -1; } else if ((size_t)rc == max_size) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: truncation might have happend with %s\n", proclink); @@ -495,14 +495,19 @@ ssize_t get_file_size (int fd) return file_size; } -dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { dyad_rc_t rc = DYAD_RC_OK; - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Applies an exclusive lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Applies an exclusive lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto excl_flock_end; @@ -511,28 +516,37 @@ dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, lock->l_whence = SEEK_SET; lock->l_start = 0; lock->l_len = 0; - lock->l_pid = ctx->pid; //getpid(); - if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock + lock->l_pid = ctx->pid; // getpid(); + if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock DYAD_LOG_ERROR (ctx, "Cannot apply exclusive lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto excl_flock_end; } - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Exclusive lock placed on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Exclusive lock placed on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; excl_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Applies a shared lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Applies a shared lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto shared_flock_end; @@ -541,43 +555,56 @@ dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, lock->l_whence = SEEK_SET; lock->l_start = 0; lock->l_len = 0; - lock->l_pid = ctx->pid; //getpid(); - if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock + lock->l_pid = ctx->pid; // getpid(); + if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock DYAD_LOG_ERROR (ctx, "Cannot apply shared lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto shared_flock_end; } - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Shared lock placed on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Shared lock placed on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; shared_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Releases a lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Releases a lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto release_flock_end; } lock->l_type = F_UNLCK; - if (fcntl (fd, F_SETLKW, lock) == -1) { // will just unlock + if (fcntl (fd, F_SETLKW, lock) == -1) { // will just unlock DYAD_LOG_ERROR (ctx, "Cannot release lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto release_flock_end; } - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] lock lifted from fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] lock lifted from fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; release_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -597,10 +624,7 @@ int sync_containing_dir (const char* path) if (dir_fd < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, - PATH_MAX + 256, - "Failed to open directory %s\n", - containing_dir); + snprintf (errmsg, PATH_MAX + 256, "Failed to open directory %s\n", containing_dir); perror (errmsg); return -1; // exit (SYS_ERR); } diff --git a/src/dyad/utils/utils.h b/src/dyad/utils/utils.h index 80533aeb..02ab6a15 100644 --- a/src/dyad/utils/utils.h +++ b/src/dyad/utils/utils.h @@ -24,18 +24,18 @@ #if defined(__cplusplus) // #include // c++11 #include -#include #include +#include #else #include #include -#include #include +#include #endif // defined(__cplusplus) - #include -#include + #include +#include #if defined(__cplusplus) extern "C" { @@ -93,11 +93,14 @@ bool get_stat (const char* path, unsigned int max_retry, long ns_sleep); ssize_t get_file_size (int fd); -dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock); -dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock); -dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock); #if DYAD_SYNC_DIR diff --git a/src/dyad/wrapper/flux_barrier.c b/src/dyad/wrapper/flux_barrier.c index d55ee230..e44e8266 100644 --- a/src/dyad/wrapper/flux_barrier.c +++ b/src/dyad/wrapper/flux_barrier.c @@ -8,15 +8,13 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ - +#include #include #include #include #include -#include #include #include -#include #if DYAD_FULL_DEBUG #define IPRINTF DPRINTF #define IPRINTF_DEFINED @@ -94,12 +92,12 @@ int main (int argc, char** argv) flux_future_destroy (fb); -// struct timespec t_now; -// clock_gettime (CLOCK_MONOTONIC_RAW, &t_now); -// char tbuf[100]; -// strftime (tbuf, sizeof (tbuf), "%D %T", gmtime (&(t_now.tv_sec))); + // struct timespec t_now; + // clock_gettime (CLOCK_MONOTONIC_RAW, &t_now); + // char tbuf[100]; + // strftime (tbuf, sizeof (tbuf), "%D %T", gmtime (&(t_now.tv_sec))); -// FLUX_LOG_INFO ("Synchronized at %s.%09ld", tbuf, t_now.tv_nsec); + // FLUX_LOG_INFO ("Synchronized at %s.%09ld", tbuf, t_now.tv_nsec); return EXIT_SUCCESS; }