move shuffle into tests

CMakeLists.txt

@@ -283,6 +283,7 @@ endfunction()
 include(DYADUtils)
 include_directories(${CMAKE_SOURCE_DIR}/include)  # public header
 add_subdirectory(src/dyad)
+add_subdirectory(tests/shuffle)
 #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order
 
 

src/dyad/utils/read_all.h

@@ -32,7 +32,7 @@ ssize_t
 read_all (int fd, void **bufp);
 
 #if defined(__cplusplus)
-};
+}
 #endif  // defined(__cplusplus)
 
 #endif /* DYAD_UTILS_READ_ALL_H */

tests/shuffle/CMakeLists.txt

@@ -0,0 +1,34 @@
+find_package(MPI)
+
+if (MPI_FOUND)
+    set(DYAD_TEST_SHUFFLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/shuffle.cpp
+                              ${CMAKE_CURRENT_SOURCE_DIR}/worker.cpp)
+    set(DYAD_TEST_SHUFFLE_PRIVATE_HEADERS  ${CMAKE_CURRENT_SOURCE_DIR}/worker.hpp
+                                           ${CMAKE_CURRENT_SOURCE_DIR}/../src/dyad/utils/utils.h
+                                           ${CMAKE_CURRENT_SOURCE_DIR}/../src/dyad/utils/read_all.h)
+    set(DYAD_TEST_SHUFFLE_PUBLIC_HEADERS)
+
+
+    add_executable(shuffle ${CMAKE_CURRENT_SOURCE_DIR}/shuffle.cpp
+                           ${CMAKE_CURRENT_SOURCE_DIR}/worker.cpp)
+    target_compile_definitions(shuffle PUBLIC DYAD_HAS_CONFIG)
+    target_include_directories(shuffle PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
+        $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/src>
+        $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>
+        $<INSTALL_INTERFACE:${DYAD_INSTALL_INCLUDE_DIR}>)
+    target_link_libraries(shuffle PUBLIC ${PROJECT_NAME}_utils MPI::MPI_CXX)
+
+
+    if(DYAD_LOGGER STREQUAL "CPP_LOGGER")
+        target_link_libraries(shuffle PRIVATE ${CPP_LOGGER_LIBRARIES})
+    endif()
+    if(DYAD_PROFILER STREQUAL "DLIO_PROFILER")
+        target_link_libraries(shuffle PRIVATE ${DLIO_PROFILER_LIBRARIES})
+    endif()
+
+
+    if (TARGET DYAD_CXX_FLAGS_werror)
+        target_link_libraries(shuffle PRIVATE DYAD_CXX_FLAGS_werror)
+    endif ()
+endif (MPI_FOUND)

use_cases/shuffle/README.md → tests/shuffle/README.md

@@ -1,11 +1,11 @@
-Use case demonstration for input file shuffling with deep learning. 
+Demonstration of input file shuffling in deep learning. 
 
 Deep learning training often requires randomizing the order of input samples at
 each epoch. In distributed or parallel training where each worker consumes a
 subset of samples, the set of samples to read changes for each worker at every
 epoch due to the randomization. 
 In this demo, we assume that each sample is read from a unique file such as an
-image file. We also assume that the amount of input samples are is large to fit
+image file. We also assume that the amount of input samples is too large to fit
 in a single local storage of a worker. Therefore, input data resides on a shared
 storage. 
 With DYAD, a worker will be able to avoid loading files from a shared storage at
@@ -15,5 +15,5 @@ loads files from its partition into its local storage.
 We currently mimic this by creating a set of files under a DYAD-managed
 directory on the local storage. Each worker can access any sample as if it exists
 locally. 
-As the capacity of a local storage reaches its limit, some files will have to be
-randomly evicted, especially when loading a new file.
+TODO: As the capacity of a local storage reaches its limit, some files will have
+to be randomly evicted, especially when loading a new file.

use_cases/shuffle/shuffle.cpp → tests/shuffle/shuffle.cpp

@@ -15,8 +15,8 @@
 #include <sstream>
 #include <vector>
 
-#include "read_all.h"
-#include "utils.h"
+#include <dyad/utils/read_all.h>
+#include <dyad/utils/utils.h>
 #include "worker.hpp"
 
 using std::cout;

use_cases/shuffle/worker.hpp → tests/shuffle/worker.hpp

@@ -1,5 +1,5 @@
-#ifndef WORKER_HPP
-#define WORKER_HPP
+#ifndef DYAD_TEST_SHUFFLE_WORKER_HPP
+#define DYAD_TEST_SHUFFLE_WORKER_HPP
 
 #include <random>
 #include <string>
@@ -51,4 +51,4 @@ class Worker
     void set_file_list ();
 };
 
-#endif  // WORKER_HPP
+#endif  // DYAD_TEST_SHUFFLE_WORKER_HPP