v21: Build: port to cmake

.gitignore

@@ -27,6 +27,12 @@
 *.out
 *.app
 
+# CLion related
+.idea
+cmake-build-*
+
 results
 errors
 *_test
+build-*/
+code_doc/

CMakeLists.txt

@@ -0,0 +1,178 @@
+cmake_minimum_required(VERSION 3.12)
+project(LCI
+  VERSION 0.2.1
+  DESCRIPTION "Lightweight Communication Interface"
+  HOMEPAGE_URL "https://github.com/uiuc-hpc/LC"
+  LANGUAGES C
+)
+
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+include(TargetSourcesRelative)
+
+set(LCI_SERVER psm CACHE STRING "Fabric")
+set_property(CACHE LCI_SERVER PROPERTY STRINGS ofi ibv psm)
+option(LCI_SERVER_DEBUG "Server debug" OFF)
+option(LCI_USE_DREG "Use registration cache (only affects ibv -- only use when not-so-dynamic allocation)" ON)
+option(LCI_USE_PMI2 "Enable PMI2" OFF)
+set(LCI_EP_AR dynamic explicit CACHE STRING "Addressing mode (dynamic, explicit, immediate)")
+set(LCI_EP_CE sync cq CACHE STRING "Completion mechanism (sync, cq, am, glob)")
+set(LCI_MAX_DEV 1 CACHE STRING "Maximum number of devices")
+set(LCI_MAX_EP 8 CACHE STRING "Maximum number of EP")
+
+set(LCI_SYNC flag CACHE STRING "Sync mechanism")
+set_property(CACHE LCI_SYNC PROPERTY STRINGS flag argobots)
+
+if(LCI_SERVER STREQUAL "ofi")
+  set(LCI_USE_SERVER_OFI ON)
+elseif(LCI_SERVER STREQUAL "ibv")
+  set(LCI_USE_SERVER_IBV ON)
+elseif(LCI_SERVER STREQUAL "psm")
+  set(LCI_USE_SERVER_PSM ON)
+else()
+  message(FATAL_ERROR "Fabric ${LCI_SERVER} not supported")
+endif()
+
+if("dynamic" IN_LIST LCI_EP_AR)
+  set(LCI_SERVER_HAS_DYN ON)
+endif()
+if("explicit" IN_LIST LCI_EP_AR)
+  set(LCI_SERVER_HAS_EXP ON)
+endif()
+if("immediate" IN_LIST LCI_EP_AR)
+  set(LCI_SERVER_HAS_IMM ON)
+endif()
+
+if("sync" IN_LIST LCI_EP_CE)
+  set(LCI_SERVER_HAS_SYNC ON)
+endif()
+if("cq" IN_LIST LCI_EP_CE)
+  set(LCI_SERVER_HAS_CQ ON)
+endif()
+if("am" IN_LIST LCI_EP_CE)
+  set(LCI_SERVER_HAS_AM ON)
+endif()
+if("glob" IN_LIST LCI_EP_CE)
+  set(LCI_SERVER_HAS_GLOB ON)
+endif()
+
+if(LCI_SYNC STREQUAL "flag")
+  set(LCI_SYNC_FLAG ON)
+elseif(LCI_SYNC STREQUAL "argobots")
+  set(LCI_SYNC_ABT ON)
+else()
+  message(FATAL_ERROR "Sync mechanism ${LCI_SYNC} not supported")
+endif()
+
+set(USE_AFFI ON CACHE BOOL "CPU affinity")
+mark_as_advanced(USE_AFFI)
+if(USE_AFFI)
+  set(AFF_DEBUG ON)
+  add_compile_definitions(USE_AFFI AFF_DEBUG)
+endif()
+
+set(USE_DREG ${LCI_USE_DREG})
+
+#NOTE(danghvu): These numbers are tweaked for performance and some alignment.
+#Update at our own risk.
+if(LCI_USE_SERVER_OFI)
+  set(LCI_PACKET_SIZE "(32 * 1024 + 4096)")
+else()
+  set(LCI_PACKET_SIZE "(8 * 1024 + 4096)")
+endif()
+
+if(LCI_USE_SERVER_PSM)
+  set(LCI_MAX_INLINE "1024")
+else()
+  set(LCI_MAX_INLINE "64")
+endif()
+
+set(LCI_PACKET_SIZE ${LCI_PACKET_SIZE} CACHE STRING "Size of packet")
+set(LCI_MAX_INLINE ${LCI_MAX_INLINE} CACHE STRING "Max inline message size")
+set(LCI_DEV_MEM_SIZE "(64*1024*1024)" CACHE STRING "Size of device memory")
+set(LCI_SERVER_MAX_RCVS 64 CACHE STRING "Max posted recvs")
+set(LCI_SERVER_NUM_PKTS 1024 CACHE STRING "Number of packets")
+set(LCI_CACHE_LINE 64 CACHE STRING "Size of cache line (bytes)")
+mark_as_advanced(
+  LCI_PACKET_SIZE
+  LCI_MAX_INLINE
+  LCI_DEV_MEM_SIZE
+  LCI_SERVER_MAX_RCVS
+  LCI_SERVER_NUM_PKTS
+  LCI_CACHE_LINE
+)
+
+
+
+
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_package(Threads REQUIRED)
+
+string(TOUPPER ${LCI_SERVER} FABRIC_PREFER)
+find_package(Fabric REQUIRED)
+
+find_package(Argobots)
+
+add_library(lci-obj OBJECT)
+set_target_properties(lci-obj PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  C_VISIBILITY_PRESET       hidden
+  C_STANDARD                99
+  C_EXTENSIONS              ON
+)
+target_compile_definitions(lci-obj PRIVATE _GNU_SOURCE)
+target_include_directories(lci-obj PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+target_link_libraries(lci-obj PUBLIC
+  Threads::Threads
+  Fabric::Fabric
+  $<$<BOOL:${LCI_SYNC_ABT}>:Argobots::Argobots>
+)
+
+add_library(lci_shared SHARED)
+add_library(lci_static STATIC)
+target_link_libraries(lci_shared PUBLIC lci-obj)
+target_link_libraries(lci_static PUBLIC lci-obj)
+set_target_properties(lci_shared lci_static PROPERTIES OUTPUT_NAME lci)
+
+add_subdirectory(src)
+add_subdirectory(examples)
+
+
+
+write_basic_package_version_file(
+  "${CMAKE_CURRENT_BINARY_DIR}/LCIConfigVersion.cmake"
+  COMPATIBILITY ExactVersion
+)
+configure_package_config_file(LCIConfig.cmake.in LCIConfig.cmake
+  INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake"
+  PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR
+)
+set(PKGCONFIG_REQUIRES_PRIVATE ${Fabric_${FABRIC_PREFER}_PC_Requires})
+set(PKGCONFIG_LIBS_PRIVATE ${Fabric_${FABRIC_PREFER}_PC_Libs})
+configure_file(liblci.pc.in liblci.pc @ONLY)
+
+install(TARGETS lci_shared lci_static
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+install(DIRECTORY include/
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+        FILES_MATCHING PATTERN "*.h"
+)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblci.pc"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
+)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/LCIConfig.cmake"
+              "${CMAKE_CURRENT_BINARY_DIR}/LCIConfigVersion.cmake"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake
+)
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/FindFabric.cmake"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LCI
+)
+install(PROGRAMS lcrun DESTINATION ${CMAKE_INSTALL_BINDIR})

LCIConfig.cmake.in

@@ -0,0 +1,28 @@
+@PACKAGE_INIT@
+
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/LCI")
+include(CMakeFindDependencyMacro)
+
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_dependency(Threads)
+find_dependency(Fabric)
+
+set_and_check(LCI_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
+set_and_check(LCI_SHARED_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_SHARED_LIBRARY_PREFIX@lci@CMAKE_SHARED_LIBRARY_SUFFIX@")
+set_and_check(LCI_STATIC_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_STATIC_LIBRARY_PREFIX@lci@CMAKE_STATIC_LIBRARY_SUFFIX@")
+
+add_library(LCI::Shared SHARED IMPORTED)
+set_target_properties(LCI::Shared PROPERTIES
+  IMPORTED_LOCATION ${LCI_SHARED_LIBRARY}
+)
+target_include_directories(LCI::Shared INTERFACE ${LCI_INCLUDE_DIRS})
+
+add_library(LCI::Static STATIC IMPORTED)
+set_target_properties(LCI::Static PROPERTIES
+  IMPORTED_LOCATION ${LCI_STATIC_LIBRARY}
+)
+target_include_directories(LCI::Static INTERFACE ${LCI_INCLUDE_DIRS})
+target_link_libraries(LCI::Static INTERFACE Threads::Threads Fabric::@FABRIC_PREFER@)
+
+check_required_components(LCI)

Makefile

@@ -86,14 +86,17 @@ endif
 
 ifeq ($(LCI_SERVER), ofi)
 	CFLAGS += -DLCI_USE_SERVER_OFI -DAFF_DEBUG
+	LDFLAGS += -lfabric
 endif
 
 ifeq ($(LCI_SERVER), ibv)
 	CFLAGS += -DLCI_USE_SERVER_IBV -DAFF_DEBUG -I$(IBV_DIR)/include
+	LDFLAGS += -L$(IBV_DIR)/lib -libverbs
 endif
 
 ifeq ($(LCI_SERVER), psm)
 	CFLAGS += -DLCI_USE_SERVER_PSM -DAFF_DEBUG -I$(PSM_DIR)/include
+	LDFLAGS += -L$(PSM_DIR)/lib -lpsm2
 endif
 
 SYNC_FLAG = sync_flag.o
@@ -118,19 +121,22 @@ LIBOBJ += $(OBJECTS) $(addprefix $(OBJDIR)/, $(FCONTEXT)) $(addprefix $(OBJDIR)/
 
 LIBRARY = liblci.so
 ARCHIVE = liblci.a
+PKGCONFIG = liblci.pc
 
 all: $(LIBRARY) $(ARCHIVE)
 
 install: all
 	mkdir -p $(PREFIX)/bin
 	mkdir -p $(PREFIX)/lib
+	mkdir -p $(PREFIX)/lib/pkgconfig
 	mkdir -p $(PREFIX)/include
 	cp lcrun $(PREFIX)/bin
 	cp -R include/* $(PREFIX)/include
-	cp liblwci.a $(PREFIX)/lib
-	cp liblwci.so $(PREFIX)/lib
+	cp $(ARCHIVE) $(PREFIX)/lib
+	cp $(LIBRARY) $(PREFIX)/lib
+	cp $(PKGCONFIG) $(PREFIX)/lib/pkgconfig
 
-$(OBJDIR)/%.o: $(SRCDIR)/$(notdir %.c)
+$(OBJDIR)/%.o: $(SRCDIR)/$(notdir %.c) $(SRCDIR)/include/config.h
 	$(CC) $(CFLAGS) -c $< -o $@
 
 $(LIBRARY): $(LIBOBJ)
@@ -141,12 +147,15 @@ $(ARCHIVE): $(LIBOBJ)
 	$(AR) q $(ARCHIVE) $(LIBOBJ) $(MALLOC)
 	$(RANLIB) $(ARCHIVE)
 
+$(SRCDIR)/include/config.h:: $(SRCDIR)/include/config.h.mk
+	cp $< $@
+
 mpiv.a: obj/mpiv.o
 	$(AR) q mpiv.a obj/mpiv.o $(LIBOBJ)
 	$(RANLIB) mpiv.a
 
 clean:
-	rm -rf $(LIBOBJ) $(OBJDIR)/* liblwci.a liblwci.so mpiv.a
+	rm -rf $(LIBOBJ) $(OBJDIR)/* $(SRCDIR)/include/config.h $(ARCHIVE) $(LIBRARY) mpiv.a
 	$(MAKE) clean -C examples
 	$(MAKE) clean -C benchmarks
 

cmake_modules/FindArgobots.cmake

@@ -0,0 +1,79 @@
+#[=======================================================================[.rst:
+FindArgobots
+----------
+
+Finds Argobots
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module provides the following imported targets, if found:
+
+``Argobots::Argobots``
+  The Argobots library
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+``Argobots_FOUND``
+  Set if Argobots was found
+
+``Argobots_INCLUDE_DIRS``
+  Include directories needed to use Argobots
+
+``Argobots_LIBRARIES``
+  Libraries needed to link to Argobots
+
+``Argobots_CFLAGS_OTHER``
+  Other CFLAGS needed to use Argobots
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``Argobots_INCLUDE_DIR``
+  The include directory for Argobots
+
+``Argobots_LIBRARY``
+  Path to the library for Argobots
+
+#]=======================================================================]
+
+include(FindPackageHandleStandardArgs)
+find_package(PkgConfig)
+
+pkg_check_modules(_Argobots_PC QUIET argobots)
+find_path(Argobots_INCLUDE_DIR
+          NAMES "abt.h"
+          PATHS ${_Argobots_PC_INCLUDE_DIRS}
+)
+find_library(Argobots_LIBRARY
+             NAMES "abt"
+             PATHS ${_Argobots_PC_LIBRARY_DIRS}
+)
+set(Argobots_VERSION ${_Argobots_PC_VERSION})
+
+find_package_handle_standard_args(Argobots
+  REQUIRED_VARS
+    Argobots_INCLUDE_DIR
+    Argobots_LIBRARY
+  VERSION_VAR Argobots_VERSION
+)
+
+if(Argobots_FOUND)
+  set(Argobots_INCLUDE_DIRS ${Argobots_INCLUDE_DIR})
+  set(Argobots_LIBRARIES ${Argobots_LIBRARY})
+  set(Argobots_CFLAGS_OTHER ${_Argobots_PC_CFLAGS_OTHER})
+
+  if(NOT TARGET Argobots::Argobots)
+    add_library(Argobots::Argobots UNKNOWN IMPORTED)
+    set_target_properties(Argobots::Argobots PROPERTIES
+      IMPORTED_LOCATION "${Argobots_LIBRARY}"
+      INTERFACE_INCLUDE_DIRECTORIES "{Argobots_INCLUDE_DIR}"
+      INTERFACE_COMPILE_OPTIONS "${_Argobots_PC_CFLAGS_OTHER}"
+    )
+  endif()
+
+  mark_as_advanced(Argobots_INCLUDE_DIR Argobots_LIBRARY)
+endif()