diff --git a/CMakeLists.txt b/CMakeLists.txt index c3610a3b..f11eee8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -283,6 +283,7 @@ endfunction() include(DYADUtils) include_directories(${CMAKE_SOURCE_DIR}/include) # public header add_subdirectory(src/dyad) +add_subdirectory(tests/shuffle) #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order diff --git a/src/dyad/utils/read_all.h b/src/dyad/utils/read_all.h index a7acd4ce..a174bef5 100644 --- a/src/dyad/utils/read_all.h +++ b/src/dyad/utils/read_all.h @@ -32,7 +32,7 @@ ssize_t read_all (int fd, void **bufp); #if defined(__cplusplus) -}; +} #endif // defined(__cplusplus) #endif /* DYAD_UTILS_READ_ALL_H */ diff --git a/tests/shuffle/CMakeLists.txt b/tests/shuffle/CMakeLists.txt new file mode 100644 index 00000000..0ab69460 --- /dev/null +++ b/tests/shuffle/CMakeLists.txt @@ -0,0 +1,34 @@ +find_package(MPI) + +if (MPI_FOUND) + set(DYAD_TEST_SHUFFLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/shuffle.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/worker.cpp) + set(DYAD_TEST_SHUFFLE_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/worker.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/dyad/utils/utils.h + ${CMAKE_CURRENT_SOURCE_DIR}/../src/dyad/utils/read_all.h) + set(DYAD_TEST_SHUFFLE_PUBLIC_HEADERS) + + + add_executable(shuffle ${CMAKE_CURRENT_SOURCE_DIR}/shuffle.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/worker.cpp) + target_compile_definitions(shuffle PUBLIC DYAD_HAS_CONFIG) + target_include_directories(shuffle PUBLIC + $ + $ + $ + $) + target_link_libraries(shuffle PUBLIC ${PROJECT_NAME}_utils MPI::MPI_CXX) + + + if(DYAD_LOGGER STREQUAL "CPP_LOGGER") + target_link_libraries(shuffle PRIVATE ${CPP_LOGGER_LIBRARIES}) + endif() + if(DYAD_PROFILER STREQUAL "DLIO_PROFILER") + target_link_libraries(shuffle PRIVATE ${DLIO_PROFILER_LIBRARIES}) + endif() + + + if (TARGET DYAD_CXX_FLAGS_werror) + target_link_libraries(shuffle PRIVATE DYAD_CXX_FLAGS_werror) + endif () +endif (MPI_FOUND) diff --git a/use_cases/shuffle/README.md b/tests/shuffle/README.md similarity index 76% rename from use_cases/shuffle/README.md rename to tests/shuffle/README.md index e7d3709a..4d6efe93 100644 --- a/use_cases/shuffle/README.md +++ b/tests/shuffle/README.md @@ -1,11 +1,11 @@ -Use case demonstration for input file shuffling with deep learning. +Demonstration of input file shuffling in deep learning. Deep learning training often requires randomizing the order of input samples at each epoch. In distributed or parallel training where each worker consumes a subset of samples, the set of samples to read changes for each worker at every epoch due to the randomization. In this demo, we assume that each sample is read from a unique file such as an -image file. We also assume that the amount of input samples are is large to fit +image file. We also assume that the amount of input samples is too large to fit in a single local storage of a worker. Therefore, input data resides on a shared storage. With DYAD, a worker will be able to avoid loading files from a shared storage at @@ -15,5 +15,5 @@ loads files from its partition into its local storage. We currently mimic this by creating a set of files under a DYAD-managed directory on the local storage. Each worker can access any sample as if it exists locally. -As the capacity of a local storage reaches its limit, some files will have to be -randomly evicted, especially when loading a new file. +TODO: As the capacity of a local storage reaches its limit, some files will have +to be randomly evicted, especially when loading a new file. diff --git a/use_cases/shuffle/shuffle.cpp b/tests/shuffle/shuffle.cpp similarity index 98% rename from use_cases/shuffle/shuffle.cpp rename to tests/shuffle/shuffle.cpp index 5cc087aa..1f51fdbf 100644 --- a/use_cases/shuffle/shuffle.cpp +++ b/tests/shuffle/shuffle.cpp @@ -15,8 +15,8 @@ #include #include -#include "read_all.h" -#include "utils.h" +#include +#include #include "worker.hpp" using std::cout; diff --git a/use_cases/shuffle/worker.cpp b/tests/shuffle/worker.cpp similarity index 100% rename from use_cases/shuffle/worker.cpp rename to tests/shuffle/worker.cpp diff --git a/use_cases/shuffle/worker.hpp b/tests/shuffle/worker.hpp similarity index 91% rename from use_cases/shuffle/worker.hpp rename to tests/shuffle/worker.hpp index 29596d56..49848eff 100644 --- a/use_cases/shuffle/worker.hpp +++ b/tests/shuffle/worker.hpp @@ -1,5 +1,5 @@ -#ifndef WORKER_HPP -#define WORKER_HPP +#ifndef DYAD_TEST_SHUFFLE_WORKER_HPP +#define DYAD_TEST_SHUFFLE_WORKER_HPP #include #include @@ -51,4 +51,4 @@ class Worker void set_file_list (); }; -#endif // WORKER_HPP +#endif // DYAD_TEST_SHUFFLE_WORKER_HPP diff --git a/use_cases/shuffle/Makefile b/use_cases/shuffle/Makefile deleted file mode 100644 index 0103241e..00000000 --- a/use_cases/shuffle/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -ifeq ($(DYAD_SRC_DIR),) - DYAD_SRC_DIR = ../../src -endif - -DYAD_OPTIONS += -g -O3 -Wall -fPIC -Wno-stringop-truncation -Wno-stringop-overflow - -ifeq ($(MPICXX),) - MPICXX = mpicxx -endif - -C_VER=-std=c11 -CXX_VER=-std=c++11 -CXX11_ABI=-D_GLIBCXX_USE_CXX11_ABI=0 - -CFLAGS += -D_POSIX_C_SOURCE=200112L $(DYAD_OPTIONS) \ - -I./ -I$(DYAD_SRC_DIR)/common -I$(DYAD_SRC_DIR)/modules -CXXFLAGS += -D_POSIX_C_SOURCE=200112L $(DYAD_OPTIONS) \ - -I./ -I$(DYAD_SRC_DIR)/common -I$(DYAD_SRC_DIR)/modules \ - $(CXX_VER) $(CXX11_ABI) -LDFLAGS = -HEADERS = $(DYAD_SRC_DIR)/modules/read_all.h \ - $(DYAD_SRC_DIR)/common/dyad.h \ - $(DYAD_SRC_DIR)/common/utils.h -OBJS = worker.o read_all.o utils.o - -all: shuffle utils.o read_all.o worker.o - -shuffle: shuffle.cpp $(OBJS) $(HEADERS) - $(MPICXX) $(CXXFLAGS) $< $(OBJS) -o $@ $(LDFLAGS) - -utils.o: $(DYAD_SRC_DIR)/common/utils.c $(DYAD_SRC_DIR)/common/utils.h $(DYAD_SRC_DIR)/common/dyad.h - $(CC) $(CFLAGS) $< -c - -read_all.o: $(DYAD_SRC_DIR)/modules/read_all.c $(DYAD_SRC_DIR)/modules/read_all.h - $(CC) $(CFLAGS) $< -c - -worker.o: worker.cpp worker.hpp - $(CXX) $(CXXFLAGS) $< -c - - -.PHONY: clean - -clean: - @rm -f *.o *.so *breakpoint* *.core - @rm -f shuffle