Build: port to cmake

uiuc-hpc · Apr 7, 2020 · 4b5cb1a · 4b5cb1a
1 parent 20a0d36
commit 4b5cb1a
Show file tree

Hide file tree

Showing 22 changed files with 479 additions and 7 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,157 @@
+cmake_minimum_required(VERSION 3.13)
+project(LCI
+  VERSION 0.2.0
+  DESCRIPTION "Lightweight Communication Interface"
+  HOMEPAGE_URL "https://github.com/uiuc-hpc/LC"
+  LANGUAGES C
+)
+
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+
+set(LC_SERVER psm CACHE STRING "Fabric")
+set_property(CACHE LC_SERVER PROPERTY STRINGS ofi ibv psm)
+option(LC_SERVER_DEBUG "Server debug" OFF)
+option(LC_USE_DREG "Use registration cache (only affects ibv -- only use when not-so-dynamic allocation)" ON)
+set(LC_EP_AR dynamic explicit CACHE STRING "Addressing mode (dynamic, explicit, immediate)")
+set(LC_EP_CE sync cq CACHE STRING "Completion mechanism (sync, cq, am, glob)")
+set(LC_MAX_DEV 1 CACHE STRING "Maximum number of devices")
+set(LC_MAX_EP 8 CACHE STRING "Maximum number of EP")
+
+if(LC_SERVER STREQUAL "ofi")
+  set(LC_USE_SERVER_OFI ON)
+elseif(LC_SERVER STREQUAL "ibv")
+  set(LC_USE_SERVER_IBV ON)
+elseif(LC_SERVER STREQUAL "psm")
+  set(LC_USE_SERVER_PSM ON)
+else()
+  message(FATAL_ERROR "Fabric ${LC_SERVER} not supported")
+endif()
+
+if("dynamic" IN_LIST LC_EP_AR)
+  set(LC_SERVER_HAS_DYN ON)
+endif()
+if("explicit" IN_LIST LC_EP_AR)
+  set(LC_SERVER_HAS_EXP ON)
+endif()
+if("immediate" IN_LIST LC_EP_AR)
+  set(LC_SERVER_HAS_IMM ON)
+endif()
+
+if("sync" IN_LIST LC_EP_CE)
+  set(LC_SERVER_HAS_SYNC ON)
+endif()
+if("cq" IN_LIST LC_EP_CE)
+  set(LC_SERVER_HAS_CQ ON)
+endif()
+if("am" IN_LIST LC_EP_CE)
+  set(LC_SERVER_HAS_AM ON)
+endif()
+if("glob" IN_LIST LC_EP_CE)
+  set(LC_SERVER_HAS_GLOB ON)
+endif()
+
+set(USE_AFFI ON CACHE BOOL "CPU affinity")
+mark_as_advanced(USE_AFFI)
+if(USE_AFFI)
+  set(AFF_DEBUG ON)
+  add_compile_definitions(USE_AFFI AFF_DEBUG)
+endif()
+
+set(USE_DREG ${LC_USE_DREG})
+
+#NOTE(danghvu): These numbers are tweaked for performance and some alignment.
+#Update at our own risk.
+if(LC_USE_SERVER_OFI)
+  set(LC_PACKET_SIZE "(32 * 1024 + 4096)")
+else()
+  set(LC_PACKET_SIZE "(8 * 1024 + 4096)")
+endif()
+set(LC_PACKET_SIZE ${LC_PACKET_SIZE} CACHE STRING "Size of packet")
+set(LC_MAX_INLINE 128 CACHE STRING "Max inline message size")
+set(LC_DEV_MEM_SIZE "(8*1024*1024)" CACHE STRING "Size of device memory")
+set(LC_SERVER_MAX_RCVS 64 CACHE STRING "Max posted recvs")
+set(LC_SERVER_NUM_PKTS 1024 CACHE STRING "Number of packets")
+set(LC_CACHE_LINE 64 CACHE STRING "Size of cache line (bytes)")
+mark_as_advanced(
+  LC_PACKET_SIZE
+  LC_MAX_INLINE
+  LC_DEV_MEM_SIZE
+  LC_SERVER_MAX_RCVS
+  LC_SERVER_NUM_PKTS
+  LC_CACHE_LINE
+)
+
+
+
+
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_package(Threads REQUIRED)
+
+string(TOUPPER ${LC_SERVER} FABRIC_PREFER)
+find_package(Fabric REQUIRED)
+
+
+
+add_library(lci-obj OBJECT)
+set_target_properties(lci-obj PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  C_VISIBILITY_PRESET       hidden
+  C_STANDARD                99
+  C_EXTENSIONS              ON
+)
+target_compile_definitions(lci-obj PRIVATE _GNU_SOURCE)
+target_include_directories(lci-obj PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+target_link_libraries(lci-obj PRIVATE
+  Threads::Threads
+  Fabric::Fabric
+)
+
+add_library(lci_shared SHARED)
+add_library(lci_static STATIC)
+target_link_libraries(lci_shared PUBLIC lci-obj)
+target_link_libraries(lci_static PUBLIC lci-obj)
+set_target_properties(lci_shared lci_static PROPERTIES OUTPUT_NAME lci)
+
+add_subdirectory(src)
+add_subdirectory(examples)
+
+
+
+write_basic_package_version_file(
+  "${CMAKE_CURRENT_BINARY_DIR}/LCIConfigVersion.cmake"
+  COMPATIBILITY ExactVersion
+)
+configure_package_config_file(LCIConfig.cmake.in LCIConfig.cmake
+  INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake"
+  PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR
+)
+set(PKGCONFIG_REQUIRES_PRIVATE ${Fabric_${FABRIC_PREFER}_PC_Requires})
+set(PKGCONFIG_LIBS_PRIVATE ${Fabric_${FABRIC_PREFER}_PC_Libs})
+configure_file(liblci.pc.in liblci.pc @ONLY)
+
+install(TARGETS lci_shared lci_static
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+install(DIRECTORY include/
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+        FILES_MATCHING PATTERN "*.h"
+)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblci.pc"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
+)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/LCIConfig.cmake"
+              "${CMAKE_CURRENT_BINARY_DIR}/LCIConfigVersion.cmake"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake
+)
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/FindFabric.cmake"
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LCI
+)
+install(PROGRAMS lcrun DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/LCIConfig.cmake.in b/LCIConfig.cmake.in
@@ -0,0 +1,28 @@
+@PACKAGE_INIT@
+
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/LCI")
+include(CMakeFindDependencyMacro)
+
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_dependency(Threads)
+find_dependency(Fabric)
+
+set_and_check(LCI_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
+set_and_check(LCI_SHARED_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_SHARED_LIBRARY_PREFIX@lci@CMAKE_SHARED_LIBRARY_SUFFIX@")
+set_and_check(LCI_STATIC_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_STATIC_LIBRARY_PREFIX@lci@CMAKE_STATIC_LIBRARY_SUFFIX@")
+
+add_library(LCI::Shared SHARED IMPORTED)
+set_target_properties(LCI::Shared PROPERTIES
+  IMPORTED_LOCATION ${LCI_SHARED_LIBRARY}
+)
+target_include_directories(LCI::Shared INTERFACE ${LCI_INCLUDE_DIRS})
+
+add_library(LCI::Static STATIC IMPORTED)
+set_target_properties(LCI::Static PROPERTIES
+  IMPORTED_LOCATION ${LCI_STATIC_LIBRARY}
+)
+target_include_directories(LCI::Static INTERFACE ${LCI_INCLUDE_DIRS})
+target_link_libraries(LCI::Static INTERFACE Threads::Threads Fabric::@FABRIC_PREFER@)
+
+check_required_components(LCI)
diff --git a/Makefile b/Makefile
@@ -127,7 +127,7 @@ install: all
 	cp $(LIBRARY) $(PREFIX)/lib
 	cp $(PKGCONFIG) $(PREFIX)/lib/pkgconfig
 
-$(OBJDIR)/%.o: $(SRCDIR)/$(notdir %.c)
+$(OBJDIR)/%.o: $(SRCDIR)/$(notdir %.c) $(SRCDIR)/include/config.h
 	$(CC) $(CFLAGS) -c $< -o $@
 
 $(LIBRARY): $(LIBOBJ)
@@ -138,12 +138,15 @@ $(ARCHIVE): $(LIBOBJ)
 	$(AR) q $(ARCHIVE) $(LIBOBJ) $(MALLOC)
 	$(RANLIB) $(ARCHIVE)
 
+$(SRCDIR)/include/config.h:: $(SRCDIR)/include/config.h.mk
+	cp $< $@
+
 mpiv.a: obj/mpiv.o
 	$(AR) q mpiv.a obj/mpiv.o $(LIBOBJ)
 	$(RANLIB) mpiv.a
 
 clean:
-	rm -rf $(LIBOBJ) $(OBJDIR)/* $(ARCHIVE) $(LIBRARY) mpiv.a
+	rm -rf $(LIBOBJ) $(OBJDIR)/* $(SRCDIR)/include/config.h $(ARCHIVE) $(LIBRARY) mpiv.a
 
 tests:
 	$(MAKE) -C tests && ./tests/all_test
diff --git a/cmake_modules/FindFabric.cmake b/cmake_modules/FindFabric.cmake
@@ -0,0 +1,144 @@
+#[=======================================================================[.rst:
+FindFabric
+----------
+
+Finds a low-level fabric library, such as libfabric, libibverbs, or libpsm2.
+
+Components
+^^^^^^^^^^
+
+Each library can be specified as a component.
+The following components are supported: OFI, IBV, and PSM.
+By default, this module will search for all components.
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module provides the following imported targets, if found:
+
+``Fabric::OFI``
+  The OpenFabrics Interfaces user-space library
+
+``Fabric::IBV``
+  The InfiniBand Verbs library
+
+``Fabric::PSM``
+  The Intel Performance Scaled Messaging 2 library
+
+``Fabric::Fabric``
+  INTERFACE IMPORTED library that links to ``Fabric::${FABRIC_PREFER}``
+  if the caller sets FABRIC_PREFER
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+``FABRIC_FOUND``
+  If components have been specified,
+  this will be set only if all components have been found;
+  otherwise, this is set if any fabric was found.
+
+This module will set the following variables per fabric,
+where ``fabric`` is one of OFI, IBV, or PSM:
+
+``FABRIC_<fabric>_FOUND``
+  True if ``<fabric>`` was found
+
+``FABRIC_<fabric>_INCLUDE_DIRS``
+  Include directories needed to use ``<fabric>``
+
+``FABRIC_<fabric>_LIBRARIES``
+  Libraries needed to link to ``<fabric>``
+
+``Fabric_<fabric>_CFLAGS_OTHER``
+  Other CFLAGS needed to use ``<fabric>``
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``FABRIC_<fabric>_INCLUDE_DIR``
+  The include directory for ``<fabric>``
+
+``FABRIC_<fabric>_LIBRARY``
+  Path to the library for ``<fabric>``
+
+#]=======================================================================]
+
+include(FindPackageHandleStandardArgs)
+find_package(PkgConfig)
+
+set(_OFI_PC libfabric)
+set(_OFI_HEADER "rdma/fabric.h")
+set(_OFI_LIB fabric)
+
+set(_IBV_PC libibverbs)
+set(_IBV_HEADER "infiniband/verbs.h")
+set(_IBV_LIB ibverbs)
+
+set(_PSM_PC libpsm2)
+set(_PSM_HEADER "psm2.h")
+set(_PSM_LIB psm2)
+
+foreach(FABRIC IN ITEMS OFI IBV PSM)
+  if(NOT Fabric_FIND_COMPONENTS OR ${FABRIC} IN_LIST Fabric_FIND_COMPONENTS)
+    pkg_check_modules(_Fabric_${FABRIC}_PC QUIET ${_${FABRIC}_PC})
+    find_path(Fabric_${FABRIC}_INCLUDE_DIR
+              NAMES ${_${FABRIC}_HEADER}
+              PATHS ${_Fabric_${FABRIC}_PC_INCLUDE_DIRS}
+    )
+    find_library(Fabric_${FABRIC}_LIBRARY
+                 NAMES ${_${FABRIC}_LIB}
+                 PATHS ${_Fabric_${FABRIC}_PC_LIBRARY_DIRS}
+    )
+    set(Fabric_${FABRIC}_VERSION ${_Fabric_${FABRIC}_PC_VERSION})
+
+    find_package_handle_standard_args(Fabric_${FABRIC}
+      REQUIRED_VARS
+        Fabric_${FABRIC}_INCLUDE_DIR
+        Fabric_${FABRIC}_LIBRARY
+      VERSION_VAR Fabric_${FABRIC}_VERSION
+    )
+
+    if(Fabric_${FABRIC}_FOUND)
+      set(Fabric_${FABRIC}_INCLUDE_DIRS ${Fabric_${FABRIC}_INCLUDE_DIR})
+      set(Fabric_${FABRIC}_LIBRARIES ${Fabric_${FABRIC}_LIBRARY})
+      set(Fabric_${FABRIC}_CFLAGS_OTHER ${_Fabric_${FABRIC}_PC_CFLAGS_OTHER})
+      if(_Fabric_${FABRIC}_PC_FOUND)
+        set(Fabric_${FABRIC}_PC_Requires ${_${FABRIC}_PC})
+      else()
+        set(Fabric_${FABRIC}_PC_Libs "-l${_${FABRIC}_LIB}")
+      endif()
+
+      if(NOT TARGET Fabric::${FABRIC})
+        add_library(Fabric::${FABRIC} UNKNOWN IMPORTED)
+        set_target_properties(Fabric::${FABRIC} PROPERTIES
+          IMPORTED_LOCATION "${Fabric_${FABRIC}_LIBRARY}"
+          INTERFACE_INCLUDE_DIRECTORIES "${Fabric_${FABRIC}_INCLUDE_DIR}"
+          INTERFACE_COMPILE_OPTIONS "${_Fabric_${FABRIC}_PC_CFLAGS_OTHER}"
+        )
+      endif()
+
+      if(FABRIC STREQUAL FABRIC_PREFER AND NOT TARGET Fabric::Fabric)
+        add_library(Fabric::Fabric INTERFACE IMPORTED)
+        target_link_libraries(Fabric::Fabric INTERFACE Fabric::${FABRIC})
+      endif()
+    endif()
+
+    mark_as_advanced(Fabric_${FABRIC}_INCLUDE_DIR Fabric_${FABRIC}_LIBRARY)
+  endif()
+endforeach()
+
+unset(_FABRIC_REQUIRED_VARS)
+foreach(FABRIC IN ITEMS OFI IBV PSM)
+  if((NOT Fabric_FIND_COMPONENTS AND FABRIC_${FABRIC}_FOUND) OR
+     (FABRIC STREQUAL FABRIC_PREFER) OR
+     (FABRIC IN_LIST Fabric_FIND_COMPONENTS AND Fabric_FIND_REQUIRED_${FABRIC}))
+    list(APPEND _FABRIC_REQUIRED_VARS "FABRIC_${FABRIC}_FOUND")
+  endif()
+endforeach()
+
+find_package_handle_standard_args(Fabric
+  REQUIRED_VARS ${_FABRIC_REQUIRED_VARS}
+  HANDLE_COMPONENTS
+)