From e9a340327992b9e3f88cd2b9391eff66dfc5dfd5 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Fri, 19 Jan 2024 16:00:01 +0100 Subject: [PATCH 1/7] feat: create symlinks on cmake build --- scripts/CMakeLists.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index dbe7e1060..fdcc6ed6c 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -17,3 +17,20 @@ configure_file(CMAKE_wrapper.sh CMAKE_wrapper.sh COPYONLY) configure_file(MPI_CC_wrapper.sh MPI_CC_wrapper.sh COPYONLY) configure_file(MPI_CXX_wrapper.sh MPI_CXX_wrapper.sh COPYONLY) configure_file(MPI_LINKER_wrapper.sh MPI_LINKER_wrapper.sh COPYONLY) + +# create symlinks to files +set(DP_LOCAL_BIN_DIR "$ENV{HOME}/.local/bin") +if(EXISTS ${DP_LOCAL_BIN_DIR}) + message(STATUS "Creating symlink ${DP_LOCAL_BIN_DIR}/discopop_cc to ${CMAKE_CURRENT_BINARY_DIR}/CC_wrapper.sh") + execute_process(COMMAND ln -sf ${CMAKE_CURRENT_BINARY_DIR}/CC_wrapper.sh ${DP_LOCAL_BIN_DIR}/discopop_cc) + + message(STATUS "Creating symlink ${DP_LOCAL_BIN_DIR}/discopop_cxx to ${CMAKE_CURRENT_BINARY_DIR}/CXX_wrapper.sh") + execute_process(COMMAND ln -sf ${CMAKE_CURRENT_BINARY_DIR}/CXX_wrapper.sh ${DP_LOCAL_BIN_DIR}/discopop_cxx) + + message(STATUS "Creating symlink ${DP_LOCAL_BIN_DIR}/discopop_cmake to ${CMAKE_CURRENT_BINARY_DIR}/CMAKE_wrapper.sh") + execute_process(COMMAND ln -sf ${CMAKE_CURRENT_BINARY_DIR}/CMAKE_wrapper.sh ${DP_LOCAL_BIN_DIR}/discopop_cmake) +else() + message(WARNING "Creation of symlinks discopop_cc to ${CMAKE_CURRENT_BINARY_DIR}/CC_wrapper.sh not possible. Please create it manually.") + message(WARNING "Creation of symlinks discopop_cxx to ${CMAKE_CURRENT_BINARY_DIR}/CXX_wrapper.sh not possible. Please create it manually.") + message(WARNING "Creation of symlinks discopop_cmake to ${CMAKE_CURRENT_BINARY_DIR}/CMAKE_wrapper.sh not possible. Please create it manually.") +endif() From 5965e77e062c2824eda25a5113b95e8c1fed1904 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 23 Jan 2024 10:48:21 +0100 Subject: [PATCH 2/7] doc: updated version file --- discopop_library/global_data/version/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_library/global_data/version/VERSION b/discopop_library/global_data/version/VERSION index 94ff29cc4..944880fa1 100644 --- a/discopop_library/global_data/version/VERSION +++ b/discopop_library/global_data/version/VERSION @@ -1 +1 @@ -3.1.1 +3.2.0 From a68081cd101d114ff33330adb869116ee008c46a Mon Sep 17 00:00:00 2001 From: goerlibe <23436477+goerlibe@users.noreply.github.com> Date: Fri, 19 Jan 2024 12:16:09 +0100 Subject: [PATCH 3/7] fix(config_provider): return 0, remove unneccesary casts, mutually exclusive arguments --- discopop_library/ConfigProvider/__main__.py | 11 ++++++----- discopop_library/ConfigProvider/config_provider.py | 7 +++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/discopop_library/ConfigProvider/__main__.py b/discopop_library/ConfigProvider/__main__.py index be19f3402..51441f92e 100644 --- a/discopop_library/ConfigProvider/__main__.py +++ b/discopop_library/ConfigProvider/__main__.py @@ -21,11 +21,12 @@ def parse_args() -> ConfigProviderArguments: ) # fmt: off - parser.add_argument("-b", "--dp-build-dir", action="store_true", + mutually_exclusive = parser.add_mutually_exclusive_group() + mutually_exclusive.add_argument("-b", "--dp-build-dir", action="store_true", help="Return the path to the DiscoPoP build directory") - parser.add_argument("-s", "--dp-source-dir", action="store_true", + mutually_exclusive.add_argument("-s", "--dp-source-dir", action="store_true", help="Return the path to the DiscoPoP source directory") - parser.add_argument("--llvm-bin-dir", action="store_true", + mutually_exclusive.add_argument("--llvm-bin-dir", action="store_true", help="Return the path to the LLVM bin directory") # EXPERIMENTAL FLAGS: # fmt: on @@ -39,10 +40,10 @@ def parse_args() -> ConfigProviderArguments: ) -def main() -> str: +def main() -> None: arguments = parse_args() retval = run(arguments) - return retval + print(retval) if __name__ == "__main__": diff --git a/discopop_library/ConfigProvider/config_provider.py b/discopop_library/ConfigProvider/config_provider.py index f1e7de7b9..ea7b83ae1 100644 --- a/discopop_library/ConfigProvider/config_provider.py +++ b/discopop_library/ConfigProvider/config_provider.py @@ -6,7 +6,6 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import cast from discopop_library.ConfigProvider.ConfigProviderArguments import ConfigProviderArguments from discopop_library.ConfigProvider.assets.build_config import DP_BUILD, DP_SOURCE, LLVM_BIN_DIR # type: ignore @@ -15,10 +14,10 @@ def run(arguments: ConfigProviderArguments) -> str: """Returns the contents of the written build_config.txt""" if arguments.return_dp_build_dir: - return cast(str, DP_BUILD) # type: ignore + return DP_BUILD # type: ignore elif arguments.return_dp_source_dir: - return cast(str, DP_SOURCE) # type: ignore + return DP_SOURCE # type: ignore elif arguments.return_llvm_bin_dir: - return cast(str, LLVM_BIN_DIR) # type: ignore + return LLVM_BIN_DIR # type: ignore else: raise ValueError("No valid operation for execution configuration: \n" + str(arguments)) From 8c701037fd32b95d458c65174f0b27a1d067b3d9 Mon Sep 17 00:00:00 2001 From: goerlibe <23436477+goerlibe@users.noreply.github.com> Date: Fri, 19 Jan 2024 12:29:35 +0100 Subject: [PATCH 4/7] fix(config_provider): remove experimental argument group --- discopop_library/ConfigProvider/__main__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/discopop_library/ConfigProvider/__main__.py b/discopop_library/ConfigProvider/__main__.py index 51441f92e..2bd5c187f 100644 --- a/discopop_library/ConfigProvider/__main__.py +++ b/discopop_library/ConfigProvider/__main__.py @@ -14,11 +14,6 @@ def parse_args() -> ConfigProviderArguments: """Parse the arguments passed to the discopop_config_provider""" parser = ArgumentParser(description="DiscoPoP Config Provider") - # all flags that are not considered stable should be added to the experimental_parser - experimental_parser = parser.add_argument_group( - "EXPERIMENTAL", - "Arguments for experimental features. Experimental arguments may or may not be removed or changed in the future.", - ) # fmt: off mutually_exclusive = parser.add_mutually_exclusive_group() From 1b5b55b56788b7d15af57a93c9a5dbd40f91c361 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 24 Jan 2024 11:27:53 +0100 Subject: [PATCH 5/7] fix(library)[hotspot_loader]: add file check --- discopop_library/HostpotLoader/hostpot_loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/discopop_library/HostpotLoader/hostpot_loader.py b/discopop_library/HostpotLoader/hostpot_loader.py index 768a17efb..3fdf20295 100644 --- a/discopop_library/HostpotLoader/hostpot_loader.py +++ b/discopop_library/HostpotLoader/hostpot_loader.py @@ -29,6 +29,8 @@ def run(arguments: HotspotLoaderArguments) -> Dict[HotspotType, List[Tuple[FILEI if not os.path.exists(os.path.join(os.getcwd(), "hotspot_detection")): return result_dict + if not os.path.exists(os.path.join(os.getcwd(), "hotspot_detection", "Hotspots.json")): + return result_dict with open(os.path.join(os.getcwd(), "hotspot_detection", "Hotspots.json"), "r") as f: hotspots = json.load(f) From 91732feb4b0074dff469e74b82f5caf96c54636c Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 24 Jan 2024 09:29:56 +0100 Subject: [PATCH 6/7] cleanup(optimizer): combined -e, -x, -g into optimization levels -o --- .../discopop_optimizer/OptimizerArguments.py | 15 +++---- .../discopop_optimizer/__main__.py | 15 +++---- .../optimization/evolutionary_algorithm.py | 8 ++-- .../discopop_optimizer/optimizer.py | 43 ++++++++++--------- 4 files changed, 40 insertions(+), 41 deletions(-) diff --git a/discopop_library/discopop_optimizer/OptimizerArguments.py b/discopop_library/discopop_optimizer/OptimizerArguments.py index 255fbf1f3..fde6feefb 100644 --- a/discopop_library/discopop_optimizer/OptimizerArguments.py +++ b/discopop_library/discopop_optimizer/OptimizerArguments.py @@ -16,8 +16,6 @@ class OptimizerArguments(object): verbose: bool interactive: bool - exhaustive: bool - evolutionary: Optional[List[str]] doall_microbench_file: str reduction_microbench_file: str allow_nested_parallelism: bool @@ -25,14 +23,10 @@ class OptimizerArguments(object): system_configuration_path: str check_called_function_for_nested_parallelism: bool profiling: bool - greedy: bool + optimization_level: int + optimization_level_2_parameters: str def __post_init__(self): - # fix correct optimization method - if not self.exhaustive: - if self.evolutionary == None: - self.greedy = True - self.__validate() def __validate(self): @@ -43,4 +37,9 @@ def __validate(self): if self.reduction_microbench_file is not "None": if not os.path.isfile(self.reduction_microbench_file): raise FileNotFoundError(f"Microbenchmark file not found: {self.reduction_microbench_file}") + + # check optimization level + if self.optimization_level not in [0, 1, 2, 3]: + raise ValueError("Unknown optimization level requested: ", self.optimization_level) + pass diff --git a/discopop_library/discopop_optimizer/__main__.py b/discopop_library/discopop_optimizer/__main__.py index 6af35bccd..6f6051829 100644 --- a/discopop_library/discopop_optimizer/__main__.py +++ b/discopop_library/discopop_optimizer/__main__.py @@ -23,12 +23,10 @@ def parse_args() -> OptimizerArguments: # fmt: off parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose output.") - parser.add_argument("-x", "--exhaustive", action="store_true", - help="Enable exhaustive search. By default, a greedy search is performed.") - parser.add_argument("-e", "--evolutionary", type=str, default=None, nargs=2, metavar=("population_size", "generations"), - help="Enable evolutionary search. By default, a greedy search is performed.") - parser.add_argument("-g", "--greedy", action="store_true", - help="Enable greedy search. (Default)") + parser.add_argument("-o", type=int, default=0, help="Optimization level: 0 -> no optimization. 1 -> greedy. 2 -> evolutionary. 3 -> exhaustive") + + parser.add_argument("-opt-2-params", type=str, default=None, nargs=2, metavar=("population_size", "generations"), help="Configure parameters of the evolutionary optimization (-o2). Default: 50 5") + parser.add_argument( "--doall-microbench-file", type=str, default="None", help="Do-All microbenchmark results" @@ -61,8 +59,6 @@ def parse_args() -> OptimizerArguments: return OptimizerArguments( verbose=arguments.verbose, interactive=arguments.interactive, - exhaustive=arguments.exhaustive, - evolutionary=arguments.evolutionary, doall_microbench_file=arguments.doall_microbench_file, reduction_microbench_file=arguments.reduction_microbench_file, allow_nested_parallelism=arguments.allow_nested_parallelism, @@ -70,7 +66,8 @@ def parse_args() -> OptimizerArguments: system_configuration_path=arguments.system_configuration, check_called_function_for_nested_parallelism=arguments.check_called_function_for_nested_parallelism, profiling=arguments.profiling, - greedy=arguments.greedy, + optimization_level=arguments.o, + optimization_level_2_parameters=arguments.opt_2_params, ) diff --git a/discopop_library/discopop_optimizer/optimization/evolutionary_algorithm.py b/discopop_library/discopop_optimizer/optimization/evolutionary_algorithm.py index 6fe2e780d..65928f487 100644 --- a/discopop_library/discopop_optimizer/optimization/evolutionary_algorithm.py +++ b/discopop_library/discopop_optimizer/optimization/evolutionary_algorithm.py @@ -42,11 +42,11 @@ def perform_evolutionary_search( arguments: OptimizerArguments, optimizer_dir: str, ) -> Optional[OptimizerOutputPattern]: - if arguments.evolutionary is None: - raise ValueError("Invalid arguments for evolutionary search: " + str(arguments.evolutionary)) + if arguments.optimization_level_2_parameters is None: + arguments.optimization_level_2_parameters = ["50", "5"] ### SETTINGS - population_size = int(arguments.evolutionary[0]) - generations = int(arguments.evolutionary[1]) + population_size = int(arguments.optimization_level_2_parameters[0]) + generations = int(arguments.optimization_level_2_parameters[1]) selection_strength = 0.85 # 0.8 --> 80% of the population will be selected for the next generation crossovers = int(population_size / 10) mutations = int(population_size / 10) diff --git a/discopop_library/discopop_optimizer/optimizer.py b/discopop_library/discopop_optimizer/optimizer.py index 080dc6028..e96479af5 100644 --- a/discopop_library/discopop_optimizer/optimizer.py +++ b/discopop_library/discopop_optimizer/optimizer.py @@ -208,26 +208,29 @@ def run(arguments: OptimizerArguments): print("#", key, " ->", experiment.substitutions[key]) print() - # calculate options for easy access - available_decisions = get_available_decisions_for_functions(experiment.optimization_graph, arguments) - - # calculate costs for all combinations of decisions - if arguments.exhaustive: - best_configuration = evaluate_all_decision_combinations( - experiment, available_decisions, arguments, optimizer_dir - ) - elif arguments.greedy: - best_configuration = greedy_search(experiment, available_decisions, arguments, optimizer_dir) - elif arguments.evolutionary != None: - # perform evolutionary search - best_configuration = perform_evolutionary_search( - experiment, - available_decisions, - arguments, - optimizer_dir, - ) - else: - raise ValueError("No optimization method specified!") + # apply optimization steps if requested + best_configuration = None + if arguments.optimization_level != 0: + # calculate options for easy access + available_decisions = get_available_decisions_for_functions(experiment.optimization_graph, arguments) + + if arguments.optimization_level == 1: + best_configuration = greedy_search(experiment, available_decisions, arguments, optimizer_dir) + elif arguments.optimization_level == 2: + # perform evolutionary search + best_configuration = perform_evolutionary_search( + experiment, + available_decisions, + arguments, + optimizer_dir, + ) + # calculate costs for all combinations of decisions + elif arguments.optimization_level == 3: + best_configuration = evaluate_all_decision_combinations( + experiment, available_decisions, arguments, optimizer_dir + ) + else: + raise ValueError("No valid optimization method specified: " + str(arguments.optimization_level)) if best_configuration is not None: best_configuration = optimize_updates(experiment, best_configuration, arguments) From 6d10555310a0408a68557297128b5843f6341341 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 24 Jan 2024 09:36:42 +0100 Subject: [PATCH 7/7] fix(test)[optimizer]: optimization level flags --- test/optimizer/loop_collapse/negative/simple_1/test.py | 2 +- test/optimizer/loop_collapse/positive/simple_1/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/optimizer/loop_collapse/negative/simple_1/test.py b/test/optimizer/loop_collapse/negative/simple_1/test.py index d6f6c2e62..f7a314c32 100644 --- a/test/optimizer/loop_collapse/negative/simple_1/test.py +++ b/test/optimizer/loop_collapse/negative/simple_1/test.py @@ -33,7 +33,7 @@ def test(self): # execute DiscoPoP analysis os.chdir(".discopop") os.system("discopop_explorer") - os.system("discopop_optimizer -x") + os.system("discopop_optimizer -o3") os.chdir("..") # validate results self.validate_results(current_dir, src_dir) diff --git a/test/optimizer/loop_collapse/positive/simple_1/test.py b/test/optimizer/loop_collapse/positive/simple_1/test.py index 8666f6b75..b9fd8aad8 100644 --- a/test/optimizer/loop_collapse/positive/simple_1/test.py +++ b/test/optimizer/loop_collapse/positive/simple_1/test.py @@ -33,7 +33,7 @@ def test(self): # execute DiscoPoP analysis os.chdir(".discopop") os.system("discopop_explorer") - os.system("discopop_optimizer -x") + os.system("discopop_optimizer -o3") os.chdir("..") # validate results self.validate_results(current_dir, src_dir)