From 91ceccf7de3d576c06ac2521f5ad9603c08d85e9 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 17 Jul 2024 13:18:13 -0700 Subject: [PATCH 01/37] Initial steps for adding performance testing --- scripts/CMakeLists.txt | 9 +- scripts/lc/lcats | 130 +------------------- scripts/performance/caliper.config | 4 + scripts/performance/perftest.in | 5 + scripts/spheral-setup-venv.in | 2 + tests/functional/Hydro/Noh/Noh-planar-1d.py | 4 +- tests/performance.ats | 5 + 7 files changed, 28 insertions(+), 131 deletions(-) create mode 100644 scripts/performance/caliper.config create mode 100644 scripts/performance/perftest.in create mode 100644 tests/performance.ats diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 97a8d1edb..1144be9c7 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -39,14 +39,21 @@ if (NOT ENABLE_CXXONLY) configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/lcatstest.in" "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" - ) + ) + + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/performance/perftest.in" + "${CMAKE_CURRENT_BINARY_DIR}/perftest.sh" + ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/spheral-setup-venv.sh" "${CMAKE_CURRENT_BINARY_DIR}/spheral-env.sh" "${CMAKE_CURRENT_BINARY_DIR}/atstest.sh" "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" + "${CMAKE_CURRENT_BINARY_DIR}/perftest.sh" "${CMAKE_CURRENT_SOURCE_DIR}/lc/lcats" + "${CMAKE_CURRENT_SOURCE_DIR}/performance/caliper.config" DESTINATION "${CMAKE_INSTALL_PREFIX}/scripts" ) diff --git a/scripts/lc/lcats b/scripts/lc/lcats index f6d445732..e258cad6c 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -69,7 +69,6 @@ def createBsubFile(inCommand, inAllOptions): FILE.write("\n\n") FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -119,7 +118,6 @@ def createMsubFile(inCommand, inAllOptions): FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -166,7 +164,6 @@ def createSbatchFile(inCommand, inAllOptions): # LLNL specific FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -619,17 +616,6 @@ msubFilenameDefault= "tmpAts." + ezatsStartTime + ".job" bsubFilenameDefault= "tmpAts." + ezatsStartTime + ".job" -# All test output should be placed in the KullTest -kullTestSubDir = "KullTest" - -# unique test string combines system, time and uuid -# the whole uuid4 is overkill, just take first 8 characters -from uuid import uuid4 -uniqueSubDir = SYS_TYPE + "_" + ezatsStartTime + "_" + str(uuid4())[0:8] - -uniqueKullSubdir = os.path.join(kullTestSubDir, uniqueSubDir) - - #--------------------------------------------------------------------------- # options affecting machine settings #--------------------------------------------------------------------------- @@ -698,7 +684,6 @@ parser.add_option( "--testpath", action="store", type="string", dest="testpath", parser.add_option( "--debug-build", action="store_true", dest="debugbuild", default=False, help="assume we are testing a debug build and should skip expensive (level>=100) tests.") -#parser.disable_interspersed_args() # doesn't work for this "atsWrap -b -e bin/kull test.ats " (options, args) = parser.parse_args(sys.argv[:]) # If running in SLURM, use defaults of less nodes and pdebug partition @@ -757,20 +742,8 @@ if "--help" in atsArgs or "-h" in atsArgs or "-help" in atsArgs: sys.exit(0) -#atsArgs= string.join(atsArgs) # note: lose user-intended grouping here -tmpstring=" " # moving to python3 I had to work around the oneliner above. -print(type(tmpstring)) # normally you don't need a tmp for things like this... -print("type of atsArgs var") -print(type(atsArgs)) -atsArgsStr= tmpstring.join(atsArgs) # note: lose user-intended grouping here -print(atsArgsStr) #Should be the joined list of args (space with args) -print("value of atsArgs var before assignment of atsArgsStr") -print(atsArgs) # should be the unmodified list -atsArgs=atsArgsStr -print("type of atsArgs var before assignment of atsArgsStr") -print(type(atsArgs)) -print("value of atsArgs var after assignment of atsArgsStr") -print(atsArgs) # should be the unmodified list as string +# Convert array of strings to a string with spaces for delimiters +atsArgs = " ".join(str(x) for x in atsArgs) #--------------------------------------------------------------------------- # Added this section to allow ezats to determine an appropriate filesystem @@ -826,92 +799,6 @@ def checkFileSystem(path, timeout=30): return timeoutFunction( timeout, False, canWriteToFileSystem, path ) - -def getUniqueTestPath(): - - # Helper function that verifies a temporary file can be created on file system. - # Will also catch if file system hung. - - filesystems = {} - - # SCF LLNL machines - # BG/Q -- seq entry must be before the rzuseq entry, or rzuseq will match 'seq' and try to use 'lscratch1'. - for name in ['seq']: - filesystems[name] = ['/p/lscratch1'] - - for name in ['zin', 'max']: - filesystems[name] = ['/p/lscratch2', '/p/lustre1', '/p/lscratch1'] - - for name in ['jade', 'agate', 'mica', 'magma', 'ruby']: - filesystems[name] = ['/p/lustre2', '/p/lustre1'] - - for name in ['sierra', 'tron']: - filesystems[name] = [ '/p/gpfs1' ] - - # RZ LLNL machines - for name in ['rzgenie']: - filesystems[name] = [ '/p/lustre1' ] - - for name in ['rztopaz']: - filesystems[name] = [ '/p/lustre1' ] - - for name in ['rzansel', 'lassen']: - filesystems[name] = [ '/p/gpfs1' ] - - for name in ['rzwhippet']: - filesystems[name] = [ '/p/lustre1' ] - - # Don't use NFS filesystems with BG/Q, terrible latency issues. - - # Sandia machines - filesystems['chama'] = ['/fscratch','/gscratch'] - filesystems['glory'] = ['/fscratch','/gscratch'] - filesystems['uno'] = ['/fscratch','/gscratch'] - - # Cielo - filesystems['nid'] = ['/scratch5', '/scratch4'] - - # Check which file system to use for this machine. - from platform import node - nodeName = node() - - assert 'USER' in os.environ, "ezats: No environment variable 'USER' found." - - filesystem = None - for name in list(filesystems.keys()): - if name in nodeName: - filesystem = filesystems[name] - assert filesystem, "ezats: Could not find file system entry for '%s'" % nodeName - - print("ezats: Searching for a filesystem to use...") - print("ezats: If this process hangs, you need to manually specify a file system to use via: ezats --testpath=\"/my/output/path\"") - - # Try to access file system, if no response within 4 seconds, try next file system. - fileSystemOK = False - - for entry in filesystem: - thePath = os.path.join( entry, os.environ['USER'], kullTestSubDir, uniqueSubDir ) - print("Checking filesystem path ", thePath) - fileSystemOK = checkFileSystem( thePath ) - if not fileSystemOK: - print("ezats: Could not write to filesystem location '%s', trying next file system." % thePath) - else: - fileSystemOK = True - filesystem = thePath - break - - assert fileSystemOK, "ezats: Could not locate a write accessable file system, aborting..." - - print("Found and verified file system location: %s" % filesystem) - - return filesystem - -if options.testpath: - options.testpath = os.path.abspath(options.testpath) - assert checkFileSystem(options.testpath), "ezats: Unable to create/access test path location: %s" % options.testpath -else: - options.testpath = getUniqueTestPath() - #--------------------------------------------------------------------------- #---------------------------------------------------------- @@ -920,16 +807,6 @@ else: print("Note: the srun message 'error: ioctl(TIOCGWINSZ)' can be ignored. \n[It means the process is trying to do something that requires a tty \nbut it's not doing either a read or write.]\n") -#---------------------------------------------------------- -# if batch, add allInteractive flag -#----------------------------------------------------------- - -#if os.environ.has_key('MACHINE_TYPE'): -# print "Note: setenv MACHINE_TYPE ", os.environ['MACHINE_TYPE'] -#if os.environ.has_key('BATCH_TYPE'): -# print "Note: setenv BATCH_TYPE ", os.environ['BATCH_TYPE'] -#print - #---------------------------------------------------------- # get args to add - added threaded option to the ezatsArgs or it would be passed to ats #---------------------------------------------------------- @@ -1079,12 +956,9 @@ else: if machineSettings.options.name in ['rzwhippet_flux']: os.environ["MACHINE_TYPE"] = "flux00" os.environ["BATCH_TYPE"] = "None" - os.environ["UNIQUE_KULL_TEST_SUBDIR"] = uniqueKullSubdir if platform.processor() == 'ppc64': numProcsLine = "" - # informs srun-wrapper.sh to not use kull's forkserver as ATS will run it - os.environ["KULLINATS"] = "yes" else: numProcsLine = " -n %d" % ( machineSettings.options.numNodes* cpu_count() ) diff --git a/scripts/performance/caliper.config b/scripts/performance/caliper.config new file mode 100644 index 000000000..5cccd993e --- /dev/null +++ b/scripts/performance/caliper.config @@ -0,0 +1,4 @@ +# [aggregate-report] +CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp +CALI_TIMER_INCLUSIVE_DURATION=true +CALI_MPIREPORT_CONFIG="select max(sum#time.duration) as MAX, avg(sum#time.duration) as AVG group by prop:nested format tree" diff --git a/scripts/performance/perftest.in b/scripts/performance/perftest.in new file mode 100644 index 000000000..c2d87f4ae --- /dev/null +++ b/scripts/performance/perftest.in @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +export CALI_CONFIG_FILE=@CMAKE_INSTALL_PREFIX@/scripts/caliper.config +export CALI_CONFIG_PROFILE=aggregate-report +@CMAKE_INSTALL_PREFIX@/spheral @CMAKE_INSTALL_PREFIX@/scripts/lcats --atsExe @CMAKE_INSTALL_PREFIX@/.venv/bin/ats --keep -e @CMAKE_INSTALL_PREFIX@/spheral @SPHERAL_ATS_BUILD_CONFIG_ARGS_STRING@ "$@" diff --git a/scripts/spheral-setup-venv.in b/scripts/spheral-setup-venv.in index 05363f62f..0a0af4ff7 100644 --- a/scripts/spheral-setup-venv.in +++ b/scripts/spheral-setup-venv.in @@ -22,9 +22,11 @@ cd @CMAKE_INSTALL_PREFIX@ chmod u+x scripts/spheral-env.sh chmod u+x scripts/atstest.sh chmod u+x scripts/lcatstest.sh +chmod u+x scripts/perftest.sh cp --symbolic-link scripts/spheral-env.sh spheral &> /dev/null cp --symbolic-link scripts/atstest.sh spheral-atstest &> /dev/null cp --symbolic-link scripts/lcatstest.sh spheral-lcatstest &> /dev/null +cp --symbolic-link scripts/perftest.sh spheral-perftest &> /dev/null cd - > /dev/null echo "Byte-compiling packages in install path ..." diff --git a/tests/functional/Hydro/Noh/Noh-planar-1d.py b/tests/functional/Hydro/Noh/Noh-planar-1d.py index 0d2f7d1bd..87e3e9abc 100644 --- a/tests/functional/Hydro/Noh/Noh-planar-1d.py +++ b/tests/functional/Hydro/Noh/Noh-planar-1d.py @@ -17,8 +17,8 @@ # #ATS:t10 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre' --restartStep 20 --restartFileConstructor SidreFileIO", label="Planar Noh problem -- 1-D (serial) with Sidre") #ATS:t11 = testif(t10, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", label="Planar Noh problem -- 1-D (serial) RESTART CHECK with Sidre") -#ATS:t12 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre-parrallel' --restartStep 20 --restartFileConstructor SidreFileIO", np=2, label="Planar Noh problem -- 1-D (parallel) with Sidre") -#ATS:t13 = testif(t12, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre-parrallel' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", np=2, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre") +#ATS:t12 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre-parallel' --restartStep 20 --restartFileConstructor SidreFileIO", np=2, label="Planar Noh problem -- 1-D (parallel) with Sidre") +#ATS:t13 = testif(t12, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre-parallel' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", np=2, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre") #ATS:t14 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-spio' --restartStep 20 --restartFileConstructor SidreFileIO --SPIOFileCountPerTimeslice 1", np=6, label="Planar Noh problem -- 1-D (parallel) with Sidre (SPIO check)") #ATS:t15 = testif(t14, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-spio' --restartStep 20 --restartFileConstructor SidreFileIO --SPIOFileCountPerTimeslice 1 --restoreCycle 20 --steps 20 --checkRestart True", np=6, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre (SPIO check)") # diff --git a/tests/performance.ats b/tests/performance.ats new file mode 100644 index 000000000..803ae9dd7 --- /dev/null +++ b/tests/performance.ats @@ -0,0 +1,5 @@ +#------------------------------------------------------------------------------- +# This file contains the performance tests for Spheral +#------------------------------------------------------------------------------- + +source("functional/Hydro/Noh/Noh-cylindrical-2d.py") \ No newline at end of file From 9c51f7354fdc7e89ba2fac0937e0f4abb4153f1f Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 18 Jul 2024 10:54:15 -0700 Subject: [PATCH 02/37] Start integrating Adiak --- cmake/InstallTPLs.cmake | 8 +++++++- scripts/spack/packages/spheral/package.py | 6 +++++- src/PYB11/Utilities/Adiak.py | 14 ++++++++++++++ src/PYB11/Utilities/Utilities_PYB11.py | 8 +++++++- 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 src/PYB11/Utilities/Adiak.py diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index f36fb0f16..412fef7e2 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -67,7 +67,7 @@ endif() #----------------------------------------------------------------------------------- # Use find_package to get axom (which brings in fmt) and patch fmt -find_package(axom REQUIRED QUIET NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake) +find_package(axom REQUIRED NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake) if(axom_FOUND) list(APPEND SPHERAL_BLT_DEPENDS axom) # Add fmt library to external library list @@ -88,6 +88,12 @@ foreach(_comp ${AXOM_COMPONENTS_ENABLED}) list(APPEND SPHERAL_BLT_DEPENDS ${axom_deps}) endforeach() +# Use find_package to get adiak +find_package(adiak REQUIRED NO_DEFAULT_PATH PATHS ${adiak_DIR}/lib/cmake/adiak) +if(adiak_FOUND) + list(APPEND SPHERAL_BLT_DEPENDS adiak::adiak) +endif() + # TPLs that must be imported list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo hdf5 polytope) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 6a4ddc0bc..7b76237c4 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -57,7 +57,9 @@ class Spheral(CachedCMakePackage, CudaPackage): depends_on('axom@0.7.0 ~shared +mpi +hdf5 -lua -examples -python -fortran -umpire -raja', type='build', when='+mpi') depends_on('axom@0.7.0 ~shared ~mpi +hdf5 -lua -examples -python -fortran -umpire -raja', type='build', when='~mpi') - depends_on('caliper@2.8.0 ~shared ~adiak ~libdw ~papi ~libunwind +pic', type='build') + depends_on('caliper@2.8.0 ~shared +adiak ~libdw ~papi ~libunwind +pic', type='build') + depends_on('adiak~shared+mpi', type='build', when='+mpi') + depends_on('adiak~shared~mpi', type='build', when='~mpi') depends_on('opensubdiv@3.4.3', type='build') depends_on('polytope@0.7.3 +python', type='build') @@ -155,6 +157,8 @@ def initconfig_package_entries(self): # TPL locations entries.append(cmake_cache_path('caliper_DIR', spec['caliper'].prefix)) + entries.append(cmake_cache_path('adiak_DIR', spec['adiak'].prefix)) + entries.append(cmake_cache_path('python_DIR', spec['python'].prefix)) entries.append(cmake_cache_path('boost_DIR', spec['boost'].prefix)) diff --git a/src/PYB11/Utilities/Adiak.py b/src/PYB11/Utilities/Adiak.py new file mode 100644 index 000000000..b41ff2848 --- /dev/null +++ b/src/PYB11/Utilities/Adiak.py @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Adiak utilities +#------------------------------------------------------------------------------- +from PYB11Generator import * + +@PYB11cppname("spheral_adiak_init") +def adiak_init(): + "Initialize Adiak and run collect_all" + return "void" + +@PYB11cppname("adiak::fini") +def adiak_fini(): + "Finalize Adiak" + return "void" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 45ca9ae63..47fdbdc45 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -56,13 +56,18 @@ '"Utilities/BiQuadraticInterpolator.hh"', '"Utilities/BiCubicInterpolator.hh"', '"Utilities/uniform_random.hh"', + '"Distributed/Communicator.hh"', + '"adiak.hpp"', ''] #------------------------------------------------------------------------------- # Preamble #------------------------------------------------------------------------------- PYB11preamble += """ - +inline void spheral_adiak_init() { + adiak::init((void*) &Communicator::communicator()); + adiak::collect_all(); +} """ #------------------------------------------------------------------------------- @@ -91,6 +96,7 @@ def setGlobalFlags(): from BiCubicInterpolator import * from uniform_random import * from BuildData import * +from Adiak import * ScalarScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "double")) ScalarPairScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "std::pair")) From 6dcc396a19994d0349c348b4842e257fd3405665 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 18 Jul 2024 13:56:33 -0700 Subject: [PATCH 03/37] Added adiak_value functions and updated caliper.config --- scripts/performance/caliper.config | 4 ++-- src/PYB11/Utilities/Adiak.py | 33 +++++++++++++++++++++++++- src/PYB11/Utilities/Utilities_PYB11.py | 22 ++++++++++++++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/scripts/performance/caliper.config b/scripts/performance/caliper.config index 5cccd993e..74b9668ec 100644 --- a/scripts/performance/caliper.config +++ b/scripts/performance/caliper.config @@ -1,4 +1,4 @@ # [aggregate-report] -CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp +CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp,adiak_import CALI_TIMER_INCLUSIVE_DURATION=true -CALI_MPIREPORT_CONFIG="select max(sum#time.duration) as MAX, avg(sum#time.duration) as AVG group by prop:nested format tree" +CALI_MPIREPORT_CONFIG="select max(sum#time.duration) as MAX, avg(sum#time.duration) as AVG group by prop:nested format tree(print-globals)" diff --git a/src/PYB11/Utilities/Adiak.py b/src/PYB11/Utilities/Adiak.py index b41ff2848..c2038dd71 100644 --- a/src/PYB11/Utilities/Adiak.py +++ b/src/PYB11/Utilities/Adiak.py @@ -3,7 +3,8 @@ #------------------------------------------------------------------------------- from PYB11Generator import * -@PYB11cppname("spheral_adiak_init") +# This is defined in the Utilities_PYB11.py preamble +@PYB11cppname("Spheral::spheral_adiak_init") def adiak_init(): "Initialize Adiak and run collect_all" return "void" @@ -12,3 +13,33 @@ def adiak_init(): def adiak_fini(): "Finalize Adiak" return "void" + +@PYB11cppname("adiak::collect_all") +def adiak_collect_all(): + "Collect all default Adiak metadata" + return "void" + +adiak_categories = PYB11enum(("unset", "all", "general", "performance", "control"), + doc="Enum of Adiak categories") + +@PYB11cppname("adiak::value") +@PYB11template("ValueType") +def adiak_value(name = "std::string", + value = "%(ValueType)s", + category = ("int", + "adiak_categories::general"), + subcategory = ("std::string", '""')): + "Set a single value in Adiak with a given name" + return "bool" + +@PYB11cppname("adiak::value") +@PYB11pyname("adiak_value") +@PYB11template("ValueType") +def adiak_value2(name = "std::string", + value = "%(ValueType)s", + value2 = "%(ValueType)s", + category = ("int", + "adiak_categories::general"), + subcategory = ("std::string", '""')): + "Set a pair of values in Adiak with a given name" + return "bool" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 47fdbdc45..49312c626 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -64,9 +64,18 @@ # Preamble #------------------------------------------------------------------------------- PYB11preamble += """ +namespace Spheral { inline void spheral_adiak_init() { adiak::init((void*) &Communicator::communicator()); - adiak::collect_all(); +} + +enum adiak_categories { +unset = 0, +all, +general, +performance, +control +}; } """ @@ -753,3 +762,14 @@ def clippedVolume(poly = "const Dim<3>::FacetedVolume&", planes = "const std::vector>>&"): "Return the volume of the clipped region." return "double" + +#............................................................................... +for (value, label) in (("int", "Int"), + ("unsigned", "Unsigned"), + ("long", "Long"), + ("double", "Scalar"), + ("std::string", "String")): + exec(""" +adiak_value%(label)s = PYB11TemplateFunction(adiak_value, "%(value)s") +adiak_value2%(label)s = PYB11TemplateFunction(adiak_value2, "%(value)s", pyname="adiak_value%(label)s") +""" % {"label" : label, "value" : value}) From a33402543eae807c9cff9d31d1c5d174539ea918 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 7 Aug 2024 15:25:20 -0700 Subject: [PATCH 04/37] Redid spheral spack package logic, added logic to automatically initialize and finalize Adiak when modules are loaded or destroyed, added main timer around entire program, import Caliper using find_package, switched performance.ats with performance.py with atsExit routine, fixed build output bugs and quieted some output, changed caliper configuration to spot temporarily --- cmake/InstallTPLs.cmake | 10 ++++- cmake/spheral/SpheralInstallPythonFiles.cmake | 2 +- cmake/tpl/caliper.cmake | 1 - scripts/CMakeLists.txt | 12 +++++- scripts/performance/caliper.config | 5 +-- scripts/performance/performance.py.in | 18 ++++++++ scripts/performance/perftest.in | 4 +- scripts/spack/packages/spheral/package.py | 43 +++++++++---------- scripts/spheral-setup-venv.in | 2 +- src/Distributed/CMakeLists.txt | 11 ++++- .../{mpi_mpi4py.py => mpi_mpi4py.py.in} | 9 +++- src/PYB11/Utilities/Utilities_PYB11.py | 14 ++++++ tests/performance.ats | 5 --- 13 files changed, 97 insertions(+), 39 deletions(-) delete mode 100644 cmake/tpl/caliper.cmake create mode 100644 scripts/performance/performance.py.in rename src/Distributed/{mpi_mpi4py.py => mpi_mpi4py.py.in} (95%) delete mode 100644 tests/performance.ats diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index d01246f04..4afbc3cd0 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -86,6 +86,15 @@ if(adiak_FOUND) message("Found Adiak External Package") endif() message("-----------------------------------------------------------------------------") +# Use find_package to get caliper +if (ENABLE_TIMER) + find_package(caliper REQUIRED NO_DEFAULT_PATH PATHS ${caliper_DIR}/share/cmake/caliper) + if(caliper_FOUND) + list(APPEND SPHERAL_BLT_DEPENDS caliper) + message("Found Caliper External Package") + endif() +endif() +message("-----------------------------------------------------------------------------") find_package(RAJA REQUIRED NO_DEFAULT_PATH PATHS ${raja_DIR}) if (RAJA_FOUND) message("Found RAJA External Package.") @@ -119,7 +128,6 @@ list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo hdf5 polytope) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS aneos IF ENABLE_ANEOS) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS opensubdiv IF ENABLE_OPENSUBDIV) -blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS caliper IF ENABLE_TIMER) # Create and install target library for each external library foreach(lib ${SPHERAL_EXTERN_LIBS}) diff --git a/cmake/spheral/SpheralInstallPythonFiles.cmake b/cmake/spheral/SpheralInstallPythonFiles.cmake index 01f90b749..9294f8fd9 100644 --- a/cmake/spheral/SpheralInstallPythonFiles.cmake +++ b/cmake/spheral/SpheralInstallPythonFiles.cmake @@ -14,7 +14,7 @@ function(spheral_install_python_files) install(FILES ${ARGV} DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral) install(CODE "execute_process( \ - COMMAND ${PYTHON_EXE} -m compileall DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral \ + COMMAND ${PYTHON_EXE} -m compileall ${SPHERAL_SITE_PACKAGES_PATH}/Spheral \ WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX})") endif() diff --git a/cmake/tpl/caliper.cmake b/cmake/tpl/caliper.cmake deleted file mode 100644 index 22a8e98e4..000000000 --- a/cmake/tpl/caliper.cmake +++ /dev/null @@ -1 +0,0 @@ -set(${lib_name}_libs libcaliper.a) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 1144be9c7..2e00131d7 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -46,6 +46,11 @@ if (NOT ENABLE_CXXONLY) "${CMAKE_CURRENT_BINARY_DIR}/perftest.sh" ) + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/performance/performance.py.in" + "${CMAKE_CURRENT_BINARY_DIR}/performance/performance.py" + ) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/spheral-setup-venv.sh" "${CMAKE_CURRENT_BINARY_DIR}/spheral-env.sh" @@ -55,7 +60,12 @@ if (NOT ENABLE_CXXONLY) "${CMAKE_CURRENT_SOURCE_DIR}/lc/lcats" "${CMAKE_CURRENT_SOURCE_DIR}/performance/caliper.config" DESTINATION "${CMAKE_INSTALL_PREFIX}/scripts" - ) + ) + + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/performance/performance.py" + DESTINATION "${CMAKE_INSTALL_PREFIX}/tests" + ) install(CODE "execute_process( \ COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/spheral-setup-venv.sh \ diff --git a/scripts/performance/caliper.config b/scripts/performance/caliper.config index 74b9668ec..8ec7c9024 100644 --- a/scripts/performance/caliper.config +++ b/scripts/performance/caliper.config @@ -1,4 +1,3 @@ # [aggregate-report] -CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp,adiak_import -CALI_TIMER_INCLUSIVE_DURATION=true -CALI_MPIREPORT_CONFIG="select max(sum#time.duration) as MAX, avg(sum#time.duration) as AVG group by prop:nested format tree(print-globals)" +CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp,adiak_import,timer +CALI_MPIREPORT_CONFIG="select max(sum#time.duration.ns) as MAX, avg(sum#time.duration.ns) as AVG group by prop:nested format tree(print-globals)" diff --git a/scripts/performance/performance.py.in b/scripts/performance/performance.py.in new file mode 100644 index 000000000..6459fd4d5 --- /dev/null +++ b/scripts/performance/performance.py.in @@ -0,0 +1,18 @@ +#!/user/bin/env python3 + +import sys, os +caliper_loc = "@caliper_DIR@" +sys.path.append(os.path.join(caliper_loc, "../../../lib64/caliper")) + +import caliperreader as cr + +# Put some filler functions here +def compare_times(manager): + filtered = [test for test in manager.testlist if test.status is PASSED] + for t in filtered: + print(t) + +onExit(compare_times) +glue(keep=True) +source("functional/Hydro/Noh/Noh-cylindrical-2d.py") + diff --git a/scripts/performance/perftest.in b/scripts/performance/perftest.in index c2d87f4ae..fe830a485 100644 --- a/scripts/performance/perftest.in +++ b/scripts/performance/perftest.in @@ -1,5 +1,5 @@ #!/usr/bin/env bash export CALI_CONFIG_FILE=@CMAKE_INSTALL_PREFIX@/scripts/caliper.config -export CALI_CONFIG_PROFILE=aggregate-report -@CMAKE_INSTALL_PREFIX@/spheral @CMAKE_INSTALL_PREFIX@/scripts/lcats --atsExe @CMAKE_INSTALL_PREFIX@/.venv/bin/ats --keep -e @CMAKE_INSTALL_PREFIX@/spheral @SPHERAL_ATS_BUILD_CONFIG_ARGS_STRING@ "$@" +export CALI_CONFIG_PROFILE=spot +@CMAKE_INSTALL_PREFIX@/spheral @CMAKE_INSTALL_PREFIX@/scripts/lcats --atsExe @CMAKE_INSTALL_PREFIX@/.venv/bin/ats -e @CMAKE_INSTALL_PREFIX@/spheral @SPHERAL_ATS_BUILD_CONFIG_ARGS_STRING@ "$@" diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 7b694fca6..5e143d766 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -14,7 +14,7 @@ class Spheral(CachedCMakePackage, CudaPackage): git = "https://github.com/llnl/spheral.git" tags = ['radiuss', 'simulations', 'hydrodynamics'] - maintainers = ['mdavis36','jmikeowen'] + maintainers = ['mdavis36','jmikeowen','owen32'] # ------------------------------------------------------------------------- # VERSIONS @@ -43,31 +43,30 @@ class Spheral(CachedCMakePackage, CudaPackage): depends_on('qhull@2020.2 +pic', type='build') depends_on('m-aneos@1.0') depends_on('eigen@3.4.0', type='build') - depends_on('hdf5@1.8.19 ~mpi +hl', type='build', when='~mpi') - depends_on('hdf5@1.8.19 +mpi +hl', type='build', when='+mpi') + depends_on('hdf5@1.8.19 +hl', type='build') depends_on('silo@4.10.2 +hdf5', type='build') # Zlib fix has been merged into conduit, using develop until next release. - depends_on('conduit@0.9.1 +shared +mpi +hdf5~hdf5_compat -test ~parmetis', type='build', when='+mpi') - depends_on('conduit@0.9.1 +shared ~mpi +hdf5~hdf5_compat -test ~parmetis', type='build', when='~mpi') - depends_on('conduit@0.9.1 +shared +mpi +hdf5 -test ~parmetis', type='build', when='+mpi^hdf5@1.8.0:1.8') - depends_on('conduit@0.9.1 +shared ~mpi +hdf5 -test ~parmetis', type='build', when='~mpi^hdf5@1.8.0:1.8') - - depends_on('raja@2024.02.0 +cuda cuda_arch=70', when='+cuda') - depends_on('umpire +cuda cuda_arch=70', when='+cuda') - - depends_on('raja@2024.02.0 ~cuda', when='~cuda') - depends_on('umpire ~cuda', when='~cuda') - - depends_on('axom@0.9.0 ~shared +cuda +mpi +hdf5 -lua -examples -python -fortran', type='build', when='+mpi+cuda') - depends_on('axom@0.9.0 ~shared +cuda ~mpi +hdf5 -lua -examples -python -fortran', type='build', when='~mpi+cuda') - depends_on('axom@0.9.0 ~shared ~cuda +mpi +hdf5 -lua -examples -python -fortran', type='build', when='+mpi~cuda') - depends_on('axom@0.9.0 ~shared ~cuda ~mpi +hdf5 -lua -examples -python -fortran', type='build', when='~mpi~cuda') - - depends_on('caliper@2.8.0 ~shared +adiak ~libdw ~papi ~libunwind +pic', type='build') - depends_on('adiak~shared+mpi', type='build', when='+mpi') - depends_on('adiak~shared~mpi', type='build', when='~mpi') + depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') + depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') + depends_on('axom@0.9.0 ~shared +hdf5 -lua -examples -python -fortran', type='build') + depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') + mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] + for ctpl in mpi_tpl_list: + for mpiv in ["+mpi", "~mpi"]: + depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}") + + depends_on("raja@2024.02.0", type="build") + cuda_tpl_list = ["raja", "umpire", "axom"] + with when("+cuda"): + depends_on('caliper ~cuda', type="build") + for ctpl in cuda_tpl_list: + for val in CudaPackage.cuda_arch_values: + depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") + with when("~cuda"): + for ctpl in cuda_tpl_list: + depends_on(f"{ctpl} ~cuda", type='build') depends_on('opensubdiv@3.4.3', type='build') depends_on('polytope@0.7.3 +python', type='build') diff --git a/scripts/spheral-setup-venv.in b/scripts/spheral-setup-venv.in index 0a0af4ff7..bfb2f3ec8 100644 --- a/scripts/spheral-setup-venv.in +++ b/scripts/spheral-setup-venv.in @@ -30,6 +30,6 @@ cp --symbolic-link scripts/perftest.sh spheral-perftest &> /dev/null cd - > /dev/null echo "Byte-compiling packages in install path ..." -@CMAKE_INSTALL_PREFIX@/spheral -m compileall @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@ +@CMAKE_INSTALL_PREFIX@/spheral -m compileall -q @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@ echo "Done." diff --git a/src/Distributed/CMakeLists.txt b/src/Distributed/CMakeLists.txt index 2eea61ce9..de85329bb 100644 --- a/src/Distributed/CMakeLists.txt +++ b/src/Distributed/CMakeLists.txt @@ -78,7 +78,16 @@ if (ENABLE_MPI) waitAllWithDeadlockDetection.hh ) - set(MPIPY_FILE_NAME "mpi_mpi4py.py") + if (ENABLE_TIMER) + set(MPI_TIMER_VAR "True") + else() + set(MPI_TIMER_VAR "False") + endif() + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mpi_mpi4py.py.in" + "${CMAKE_CURRENT_BINARY_DIR}/mpi_mpi4py.py" + ) + set(MPIPY_FILE_NAME "${CMAKE_CURRENT_BINARY_DIR}/mpi_mpi4py.py") else() #---------------------------------------------------------------------------- diff --git a/src/Distributed/mpi_mpi4py.py b/src/Distributed/mpi_mpi4py.py.in similarity index 95% rename from src/Distributed/mpi_mpi4py.py rename to src/Distributed/mpi_mpi4py.py.in index e1692e256..6873c5b23 100644 --- a/src/Distributed/mpi_mpi4py.py +++ b/src/Distributed/mpi_mpi4py.py.in @@ -3,7 +3,7 @@ # # This module reproduces the pyMPI interface using mpi4py. #------------------------------------------------------------------------------- -import sys +import sys, os from SpheralTestUtilities import globalFrame # NOTE: this logic for disabling recv_mprobe seems to be necessary with newer @@ -11,6 +11,13 @@ # as supported, but seem to be broken. import mpi4py mpi4py.rc.recv_mprobe = False +if (@MPI_TIMER_VAR@): + cal_run1 = os.getenv("CALI_CONFIG_PROFILE") + cal_run2 = os.getenv("CALI_CONFIG") + if (cal_run1 or cal_run2): + print("WARNING: Relying on Caliper to call MPI Finalize") + mpi4py.rc.finalize = False + # Now go on as usual... from mpi4py import MPI diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 49312c626..2cad344eb 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -56,6 +56,7 @@ '"Utilities/BiQuadraticInterpolator.hh"', '"Utilities/BiCubicInterpolator.hh"', '"Utilities/uniform_random.hh"', + '"Utilities/Timer.hh"', '"Distributed/Communicator.hh"', '"adiak.hpp"', ''] @@ -67,6 +68,8 @@ namespace Spheral { inline void spheral_adiak_init() { adiak::init((void*) &Communicator::communicator()); + // Always collect default adiak information + adiak::collect_all(); } enum adiak_categories { @@ -79,6 +82,17 @@ } """ +PYB11modulepreamble = """ +TIME_BEGIN("main"); +Spheral::spheral_adiak_init(); + +auto atexit = py::module_::import("atexit"); +atexit.attr("register")(py::cpp_function([]() { + TIME_END("main"); + adiak::fini(); +})); +""" + #------------------------------------------------------------------------------- # Namespaces #------------------------------------------------------------------------------- diff --git a/tests/performance.ats b/tests/performance.ats deleted file mode 100644 index 803ae9dd7..000000000 --- a/tests/performance.ats +++ /dev/null @@ -1,5 +0,0 @@ -#------------------------------------------------------------------------------- -# This file contains the performance tests for Spheral -#------------------------------------------------------------------------------- - -source("functional/Hydro/Noh/Noh-cylindrical-2d.py") \ No newline at end of file From 5acd1246ca936ed518dc13a79b2864047d2e8062 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 15 Aug 2024 11:16:43 -0700 Subject: [PATCH 05/37] Updated optparse to argparse, added singleton wrapp for the caliper configmanager that is python wrapped and initialized in SpheralOptionsParser and uninitialized when module is unloaded --- .gitmodules | 1 + scripts/CMakeLists.txt | 7 -- scripts/lc/lcats | 17 +---- scripts/performance/caliper.config | 3 - scripts/performance/perftest.in | 5 -- scripts/spheral-setup-venv.in | 2 - src/CRKSPH/CRKSPHHydroBase.cc | 1 - src/CRKSPH/CRKSPHHydroBaseRZ.cc | 1 - src/CRKSPH/CRKSPHVariant.cc | 1 - src/CRKSPH/SolidCRKSPHHydroBase.cc | 1 - src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc | 1 - src/DEM/DEMBase.cc | 1 - src/Distributed/CMakeLists.txt | 11 +-- .../{mpi_mpi4py.py.in => mpi_mpi4py.py} | 9 +-- src/FSISPH/SolidFSISPHHydroBase.cc | 1 - src/PYB11/Utilities/Adiak.py | 12 +--- src/PYB11/Utilities/TimerMgr.py | 42 ++++++++++++ src/PYB11/Utilities/Utilities_PYB11.py | 20 +++++- src/SimulationControl/SpheralOptionParser.py | 68 ++++++++++--------- src/Utilities/Timer.hh | 68 ++++++++++++++++++- 20 files changed, 169 insertions(+), 103 deletions(-) delete mode 100644 scripts/performance/caliper.config delete mode 100644 scripts/performance/perftest.in rename src/Distributed/{mpi_mpi4py.py.in => mpi_mpi4py.py} (95%) create mode 100644 src/PYB11/Utilities/TimerMgr.py diff --git a/.gitmodules b/.gitmodules index d24489245..107101d1a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,4 @@ path = extern/chai url = https://github.com/llnl/chai branch = feature/ManagedSharedPtr + ignore = all diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 2e00131d7..c66d12471 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -41,11 +41,6 @@ if (NOT ENABLE_CXXONLY) "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" ) - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/performance/perftest.in" - "${CMAKE_CURRENT_BINARY_DIR}/perftest.sh" - ) - configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/performance/performance.py.in" "${CMAKE_CURRENT_BINARY_DIR}/performance/performance.py" @@ -56,9 +51,7 @@ if (NOT ENABLE_CXXONLY) "${CMAKE_CURRENT_BINARY_DIR}/spheral-env.sh" "${CMAKE_CURRENT_BINARY_DIR}/atstest.sh" "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" - "${CMAKE_CURRENT_BINARY_DIR}/perftest.sh" "${CMAKE_CURRENT_SOURCE_DIR}/lc/lcats" - "${CMAKE_CURRENT_SOURCE_DIR}/performance/caliper.config" DESTINATION "${CMAKE_INSTALL_PREFIX}/scripts" ) diff --git a/scripts/lc/lcats b/scripts/lc/lcats index 83ea7145b..eb16e4171 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -642,9 +642,6 @@ parser.add_option("--wcid", action="store", type="string", metavar="WC-ID to ass #default = machineSettings.options.bank, help = "HERT WC-ID to use for batch job.") -parser.add_option( "--threaded", action="store_true", dest="threaded", - help = "Run threaded tests. NOTE ATS must run these on login node (no salloc). ") - parser.add_option( "--nogpu", action="store_true", dest="nogpu", help = "For blueos. Filters out gpu test. Used in conjunction with threaded option.") @@ -814,18 +811,6 @@ ezatsArgs= ['addOp', 'batch', 'interactive', 'name', 'allocTime', 'atsExe', 'mac # Add glue arg to pass unique file system test path to ats toAdd= """ --glue='testpath=str("%s")' """ % options.testpath -# Add threaded arg to filter for threaded or non threaded tests. Otherwise all versions will run, and threaded will be run incorrectly -if options.threaded: - if options.nogpu: - toAdd += """ --filter="'nt' in locals()" --filter="'ngpu' not in locals()" """ - elif options.gpuonly: - toAdd += """ --filter="'nt' in locals()" --filter="'ngpu' in locals()" """ - # This version only works if we're using os.system - execv call fails because filter is split incorrectly - else: - toAdd += """ --filter="'nt' in locals()" """ -else: - toAdd += """ --filter="'nt' not in locals()" """ - if options.sanitize: toAdd += """ --filter="sanitize==1" """ @@ -982,7 +967,7 @@ else: + finalCommandToRun # + " -p " + machineSettings.options.partition + " " # Threaded tests under ats should NOT use salloc - elif not options.threaded and 'blue' not in os.environ['SYS_TYPE']: + elif 'blue' not in os.environ['SYS_TYPE']: finalCommandToRun= "salloc --exclusive " \ + " " + allocTime \ + HERT_WC_ID \ diff --git a/scripts/performance/caliper.config b/scripts/performance/caliper.config deleted file mode 100644 index 8ec7c9024..000000000 --- a/scripts/performance/caliper.config +++ /dev/null @@ -1,3 +0,0 @@ -# [aggregate-report] -CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp,adiak_import,timer -CALI_MPIREPORT_CONFIG="select max(sum#time.duration.ns) as MAX, avg(sum#time.duration.ns) as AVG group by prop:nested format tree(print-globals)" diff --git a/scripts/performance/perftest.in b/scripts/performance/perftest.in deleted file mode 100644 index fe830a485..000000000 --- a/scripts/performance/perftest.in +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -export CALI_CONFIG_FILE=@CMAKE_INSTALL_PREFIX@/scripts/caliper.config -export CALI_CONFIG_PROFILE=spot -@CMAKE_INSTALL_PREFIX@/spheral @CMAKE_INSTALL_PREFIX@/scripts/lcats --atsExe @CMAKE_INSTALL_PREFIX@/.venv/bin/ats -e @CMAKE_INSTALL_PREFIX@/spheral @SPHERAL_ATS_BUILD_CONFIG_ARGS_STRING@ "$@" diff --git a/scripts/spheral-setup-venv.in b/scripts/spheral-setup-venv.in index bfb2f3ec8..cf17c0bce 100644 --- a/scripts/spheral-setup-venv.in +++ b/scripts/spheral-setup-venv.in @@ -22,11 +22,9 @@ cd @CMAKE_INSTALL_PREFIX@ chmod u+x scripts/spheral-env.sh chmod u+x scripts/atstest.sh chmod u+x scripts/lcatstest.sh -chmod u+x scripts/perftest.sh cp --symbolic-link scripts/spheral-env.sh spheral &> /dev/null cp --symbolic-link scripts/atstest.sh spheral-atstest &> /dev/null cp --symbolic-link scripts/lcatstest.sh spheral-lcatstest &> /dev/null -cp --symbolic-link scripts/perftest.sh spheral-perftest &> /dev/null cd - > /dev/null echo "Byte-compiling packages in install path ..." diff --git a/src/CRKSPH/CRKSPHHydroBase.cc b/src/CRKSPH/CRKSPHHydroBase.cc index bbdcc0ae5..12f3394ae 100644 --- a/src/CRKSPH/CRKSPHHydroBase.cc +++ b/src/CRKSPH/CRKSPHHydroBase.cc @@ -31,7 +31,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/CRKSPHHydroBaseRZ.cc b/src/CRKSPH/CRKSPHHydroBaseRZ.cc index c032358b6..b9cc6e79a 100644 --- a/src/CRKSPH/CRKSPHHydroBaseRZ.cc +++ b/src/CRKSPH/CRKSPHHydroBaseRZ.cc @@ -29,7 +29,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/CRKSPHVariant.cc b/src/CRKSPH/CRKSPHVariant.cc index da5dad87f..32c15f995 100644 --- a/src/CRKSPH/CRKSPHVariant.cc +++ b/src/CRKSPH/CRKSPHVariant.cc @@ -48,7 +48,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/SolidCRKSPHHydroBase.cc b/src/CRKSPH/SolidCRKSPHHydroBase.cc index 25fa74825..8f4bf7539 100644 --- a/src/CRKSPH/SolidCRKSPHHydroBase.cc +++ b/src/CRKSPH/SolidCRKSPHHydroBase.cc @@ -32,7 +32,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "SolidMaterial/SolidEquationOfState.hh" diff --git a/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc b/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc index 2a281f2a6..01f5774b3 100644 --- a/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc +++ b/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc @@ -33,7 +33,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/NodeCoupling.hh" #include "SolidMaterial/SolidEquationOfState.hh" diff --git a/src/DEM/DEMBase.cc b/src/DEM/DEMBase.cc index 03867b6e2..23b579f94 100644 --- a/src/DEM/DEMBase.cc +++ b/src/DEM/DEMBase.cc @@ -25,7 +25,6 @@ #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/registerWithRedistribution.hh" diff --git a/src/Distributed/CMakeLists.txt b/src/Distributed/CMakeLists.txt index de85329bb..2eea61ce9 100644 --- a/src/Distributed/CMakeLists.txt +++ b/src/Distributed/CMakeLists.txt @@ -78,16 +78,7 @@ if (ENABLE_MPI) waitAllWithDeadlockDetection.hh ) - if (ENABLE_TIMER) - set(MPI_TIMER_VAR "True") - else() - set(MPI_TIMER_VAR "False") - endif() - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/mpi_mpi4py.py.in" - "${CMAKE_CURRENT_BINARY_DIR}/mpi_mpi4py.py" - ) - set(MPIPY_FILE_NAME "${CMAKE_CURRENT_BINARY_DIR}/mpi_mpi4py.py") + set(MPIPY_FILE_NAME "mpi_mpi4py.py") else() #---------------------------------------------------------------------------- diff --git a/src/Distributed/mpi_mpi4py.py.in b/src/Distributed/mpi_mpi4py.py similarity index 95% rename from src/Distributed/mpi_mpi4py.py.in rename to src/Distributed/mpi_mpi4py.py index 6873c5b23..e1692e256 100644 --- a/src/Distributed/mpi_mpi4py.py.in +++ b/src/Distributed/mpi_mpi4py.py @@ -3,7 +3,7 @@ # # This module reproduces the pyMPI interface using mpi4py. #------------------------------------------------------------------------------- -import sys, os +import sys from SpheralTestUtilities import globalFrame # NOTE: this logic for disabling recv_mprobe seems to be necessary with newer @@ -11,13 +11,6 @@ # as supported, but seem to be broken. import mpi4py mpi4py.rc.recv_mprobe = False -if (@MPI_TIMER_VAR@): - cal_run1 = os.getenv("CALI_CONFIG_PROFILE") - cal_run2 = os.getenv("CALI_CONFIG") - if (cal_run1 or cal_run2): - print("WARNING: Relying on Caliper to call MPI Finalize") - mpi4py.rc.finalize = False - # Now go on as usual... from mpi4py import MPI diff --git a/src/FSISPH/SolidFSISPHHydroBase.cc b/src/FSISPH/SolidFSISPHHydroBase.cc index 757d32724..62ded490b 100644 --- a/src/FSISPH/SolidFSISPHHydroBase.cc +++ b/src/FSISPH/SolidFSISPHHydroBase.cc @@ -45,7 +45,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/Timer.hh" #include "Utilities/globalBoundingVolumes.hh" diff --git a/src/PYB11/Utilities/Adiak.py b/src/PYB11/Utilities/Adiak.py index c2038dd71..06c1a9187 100644 --- a/src/PYB11/Utilities/Adiak.py +++ b/src/PYB11/Utilities/Adiak.py @@ -3,16 +3,8 @@ #------------------------------------------------------------------------------- from PYB11Generator import * -# This is defined in the Utilities_PYB11.py preamble -@PYB11cppname("Spheral::spheral_adiak_init") -def adiak_init(): - "Initialize Adiak and run collect_all" - return "void" - -@PYB11cppname("adiak::fini") -def adiak_fini(): - "Finalize Adiak" - return "void" +# adiak::init() is called automatically when this module is loaded +# adiak::fini() is called automatically when this module is destroyed @PYB11cppname("adiak::collect_all") def adiak_collect_all(): diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py new file mode 100644 index 000000000..bce824414 --- /dev/null +++ b/src/PYB11/Utilities/TimerMgr.py @@ -0,0 +1,42 @@ +#------------------------------------------------------------------------------- +# TimerMgr class +#------------------------------------------------------------------------------- +from PYB11Generator import * + +@PYB11singleton +class TimerMgr: + + "Singleton wrapper for CaliperManager. Access through TimerMgr.instance(), ie TimerMgr.instance().start()." + + @PYB11static + def instance(self): + "Access the singleton instance of the timer manager" + return "TimerMgr&" + + def timer_start(self, region_name = "std::string"): + "Start custom region Caliper timer, must have corresponding timer_end call" + return "void" + + def timer_end(self, region_name = "std::string"): + "End custom region Caliper timer" + return "void" + + def add(self, config_str = "std::string"): + "Add a Caliper configuration" + return "void" + + def default_start(self, testname = "std::string"): + "Set the spot Caliper configuration and start the manager" + return "void" + + def start(self): + "Start the Caliper configuration manager" + return "void" + + def stop(self): + "Stop the Caliper configuration manager" + return "void" + + def fini(self): + "Flush the Caliper configuration manager" + return "void" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 2cad344eb..439abbf6e 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -66,10 +66,23 @@ #------------------------------------------------------------------------------- PYB11preamble += """ namespace Spheral { + inline void spheral_adiak_init() { adiak::init((void*) &Communicator::communicator()); - // Always collect default adiak information - adiak::collect_all(); + // Always collect some curated default adiak information + adiak::adiakversion(); + adiak::user(); + adiak::uid(); + adiak::launchdate(); + adiak::workdir(); + adiak::hostname(); + adiak::clustername(); + adiak::walltime(); + adiak::cputime(); + adiak::jobsize(); + adiak::numhosts(); + adiak::hostlist(); + adiak::mpi_library_version(); } enum adiak_categories { @@ -86,10 +99,12 @@ TIME_BEGIN("main"); Spheral::spheral_adiak_init(); +// Call these routines when module is destroyed auto atexit = py::module_::import("atexit"); atexit.attr("register")(py::cpp_function([]() { TIME_END("main"); adiak::fini(); + TimerMgr::instance().fini(); })); """ @@ -120,6 +135,7 @@ def setGlobalFlags(): from uniform_random import * from BuildData import * from Adiak import * +from TimerMgr import * ScalarScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "double")) ScalarPairScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "std::pair")) diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 473447c01..008e80537 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -2,51 +2,57 @@ # Create a standard and hopefully convenient command line parser for Spheral # scripts. #------------------------------------------------------------------------------- -import optparse +import argparse from SpheralCompiledPackages import * from SpheralTestUtilities import globalFrame +from SpheralUtilities import TimerMgr def commandLine(**options): # Build a command line parser with the keyword arguments passed to us. - parser = optparse.OptionParser() + parser = argparse.ArgumentParser() for key in options: - parser.add_option("--" + key, - dest = key, - default = options[key]) - - # Add the universal options suppoted by all Spheral++ scripts. - parser.add_option("-v", "--verbose", - action = "store_true", - dest = "__verbose", - default = False, - help = "Verbose output -- print all options that were set.") - + parser.add_argument("--" + key, + dest = key, + default = options[key]) + + # Add the universal options supported by all Spheral++ scripts. + parser.add_argument("-v", "--verbose", + action = "store_true", + dest = "verbose", + default = False, + help = "Verbose output -- print all options that were set.") + parser.add_argument("--caliper-config", default="", type=str) # Evaluate the command line. - opts, args = parser.parse_args() + args = parser.parse_args() + arg_dict = vars(args) # Verbose output? - if opts.__verbose: + if args.verbose: print("All parameters set:") - for key in options: - val = eval("opts.%s" % key) - if val != options[key]: - print(" * ", key, " = ", val) - else: - print(" ", key, " = ", val) + for key, val in arg_dict.items(): + if key in options: + if val != options[key]: + print(" * ", key, " = ", val) + else: + print(" ", key, " = ", val) # Set all the variables. gd = globalFrame().f_globals - for key in options: - val = eval("opts.%s" % key) - - # The following bit of goofiness is necessary because optparse returns most - # arguments as a string, but we need the actual types here. - if type(val) != type(options[key]): - val = eval(val, gd) - - gd[key] = val - + for key, val in arg_dict.items(): + if key in options: + if (type(val) != type(options[key])): + val = eval(val, gd) + gd[key] = val + + if (args.caliper_config): + TimerMgr.instance().add(args.caliper_config) + TimerMgr.instance().start() + else: + import random, os, sys + unique_digits = ''.join(random.sample('0123456789', 4)) + testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits + TimerMgr.instance().default_start(testname) return diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 93e965cea..6d9d3bc55 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -1,15 +1,79 @@ +//---------------------------------Spheral++----------------------------------// +// Timer macros and Caliper ConfigManger singleton +// +//----------------------------------------------------------------------------// +#ifndef __Spheral_Timer__ +#define __Spheral_Timer__ // If TIMER is defined then we want timer functionality #ifdef TIMER #include "caliper/cali.h" +#include "caliper/cali-manager.h" #define TIME_FUNCTION CALI_CXX_MARK_FUNCTION #define TIME_BEGIN(regionName) CALI_MARK_BEGIN(regionName) #define TIME_END(regionName) CALI_MARK_END(regionName) #else // TIMER -// Stub TIME macros, when TIME is zero +// Stub TIME macros, when TIMER is off + #define TIME_FUNCTION #define TIME_BEGIN(regionName) #define TIME_END(regionName) -#endif // TIMER \ No newline at end of file +#endif // TIMER +// Note: This class is initialized in +// SimulationControl/SpheralOptionParser.py +class TimerMgr { +public: + static TimerMgr& instance() { + static TimerMgr theInstance; + return theInstance; + } + void timer_start(std::string regionName) { + TIME_BEGIN(regionName.c_str()); + } + void timer_end(std::string regionName) { + TIME_END(regionName.c_str()); + } +#ifdef TIMER +private: + cali::ConfigManager cali_mgr; +public: + void add(std::string config_str) { + bool test = cali_mgr.add(config_str.c_str()); + VERIFY2(test, cali_mgr.error_msg()); + } + void default_start(std::string testname) { + if (!testname.empty()) { + std::string default_config = "spot,output=" + testname + ".cali,mem.highwatermark"; + add(default_config); + start(); + } else if (Spheral::Process::getRank() == 0) { + std::cout << "WARNING: Caliper test name is empty, " + << "no Caliper configuration started" << std::endl; + } + } + void start() { + cali_mgr.start(); + } + void stop() { + cali_mgr.stop(); + } + void fini() { + cali_mgr.flush(); + } +#else + void default_start(std::string) { + } + void add(std::string) { + } + void start() { + } + void stop() { + } + void fini() { + } +#endif +}; + +#endif From 60c61cd0f484c9477cd2535b238104edd2ec2bcd Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 16 Aug 2024 14:55:22 -0700 Subject: [PATCH 06/37] Fixed bugs with TimerMgr singleton --- src/PYB11/Utilities/TimerMgr.py | 1 + src/PYB11/Utilities/Utilities_PYB11.py | 2 +- src/Utilities/Timer.hh | 8 +++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index bce824414..07d88e80e 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -9,6 +9,7 @@ class TimerMgr: "Singleton wrapper for CaliperManager. Access through TimerMgr.instance(), ie TimerMgr.instance().start()." @PYB11static + @PYB11returnpolicy("reference") def instance(self): "Access the singleton instance of the timer manager" return "TimerMgr&" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 439abbf6e..d27ed25cd 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -104,7 +104,7 @@ atexit.attr("register")(py::cpp_function([]() { TIME_END("main"); adiak::fini(); - TimerMgr::instance().fini(); + Spheral::TimerMgr::instance().fini(); })); """ diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 6d9d3bc55..66d0426c5 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -23,7 +23,13 @@ #endif // TIMER // Note: This class is initialized in // SimulationControl/SpheralOptionParser.py +namespace Spheral { class TimerMgr { +private: + TimerMgr() = default; + ~TimerMgr() { } + TimerMgr(const TimerMgr&) = delete; + TimerMgr& operator=(const TimerMgr&) = delete; public: static TimerMgr& instance() { static TimerMgr theInstance; @@ -75,5 +81,5 @@ public: } #endif }; - +} #endif From cd3f88947f67c98adbce69758eb12ca346653667 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 19 Aug 2024 09:21:54 -0700 Subject: [PATCH 07/37] Added valgrind python suppression, removed & for accessing communicator for adiak init --- scripts/devtools/valgrind_python_suppression | 163 +++++++++++++++++++ src/PYB11/Utilities/Utilities_PYB11.py | 2 +- 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 scripts/devtools/valgrind_python_suppression diff --git a/scripts/devtools/valgrind_python_suppression b/scripts/devtools/valgrind_python_suppression new file mode 100644 index 000000000..619a5f106 --- /dev/null +++ b/scripts/devtools/valgrind_python_suppression @@ -0,0 +1,163 @@ +# ************************************************************************** +# Tool Gear (www.llnl.gov/CASC/tool_gear) +# Version 2.12 Jan 14, 2009 +# Please see COPYRIGHT AND LICENSE information at the end of this file. +# ************************************************************************** +# Based on python supplied valgrind suppression file, more informative +# suppression name and tweaked for x86-64 messages. -JCG 12/8/2009 +{ + _PyObject_Free ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Free +} + +{ + _PyObject_Free ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Free +} +{ + _PyObject_Free Use of uninitialised value of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Free +} + +{ + _PyObject_Free ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + ... + fun:_PyObject_Free +} +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Realloc +} + +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Realloc +} +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Realloc +} + +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + ... + fun:_PyObject_Realloc +} +{ + _PyObject_Malloc Leak + Memcheck:Leak + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid jump condition + Memcheck:Cond + ... + fun:_PyObject_Malloc +} +{ + strdup Leak + Memcheck:Leak + ... + fun:strdup +} +{ + _PyFunction_VectorCall Leak + Memcheck:Leak + ... + fun:_PyFunction_Vectorcall +} +################################################################################ +## COPYRIGHT AND LICENSE +## +## Copyright (c) 2012, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory +## Written by John Gyllenhaal (gyllen@llnl.gov), John May (johnmay@llnl.gov), +## and Martin Schulz (schulz6@llnl.gov). +## LLNL-CODE-582512 +## OCEC-12-057 +## All rights reserved. +## This file is part of Tool Gear. +## For details, see computation.llnl.gov/casc/tool_gear. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright notice, +## this list of conditions and the disclaimer below. +## +## * Redistributions in binary form must reproduce the above copyright notice, +## this list of conditions and the disclaimer (as noted below) in the +## documentation and/or other materials provided with the distribution. +## +## * Neither the name of the LLNS/LLNL nor the names of its contributors may +## be used to endorse or promote products derived from this software without +## specific prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +## THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +## PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +## SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE +## FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +## DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +## CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +## OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +## USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## +## Additional BSD Notice +## +## 1. This notice is required to be provided under our contract with the +## U.S. Department of Energy (DOE). This work was produced at +## Lawrence Livermore National Laboratory under Contract +## No. DE-AC52-07NA27344 with the DOE. +## +## 2. Neither the United States Government nor Lawrence Livermore National +## Security, LLC nor any of their employees, makes any warranty, express +## or implied, or assumes any liability or responsibility for the accuracy, +## completeness, or usefulness of any information, apparatus, product, or +## process disclosed, or represents that its use would not infringe +## privately-owned rights. +## +## 3. Also, reference herein to any specific commercial products, process, or +## services by trade name, trademark, manufacturer or otherwise does not +## necessarily constitute or imply its endorsement, recommendation, or +## favoring by the United States Government or Lawrence Livermore National +## Security, LLC. The views and opinions of authors expressed herein do not +## necessarily state or reflect those of the United States Government or +## Lawrence Livermore National Security, LLC, and shall not be used for +## advertising or product endorsement purposes. +################################################################################# diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index d27ed25cd..3ef35e9ad 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -68,7 +68,7 @@ namespace Spheral { inline void spheral_adiak_init() { - adiak::init((void*) &Communicator::communicator()); + adiak::init((void*) Communicator::communicator()); // Always collect some curated default adiak information adiak::adiakversion(); adiak::user(); From 6b6628cbd360e5b281a2af00f3cee46570882f68 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 19 Aug 2024 11:42:10 -0700 Subject: [PATCH 08/37] Fixed inconsistency in Communicator for non-MPI builds and added pointer function to get null pointer for adiak --- src/Distributed/Communicator.hh | 10 +++++++--- src/PYB11/Utilities/Utilities_PYB11.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index 11a03fb13..19437d56d 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -9,6 +9,8 @@ #ifdef USE_MPI #include +#else +typedef int MPI_Comm; #endif namespace Spheral { @@ -21,18 +23,20 @@ public: static Communicator& instance() { static Communicator theInstance; return theInstance; } // Access the communicator. -#ifdef USE_MPI static MPI_Comm& communicator() { return instance().mCommunicator; } static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } +#ifdef USE_MPI + static MPI_Comm* comm_ptr() { return &(instance().mCommunicator); } #else - static int communicator() { return 0; } - static void communicator(int&) {} + static MPI_Comm* comm_ptr() { return nullptr; } #endif private: //------------------------===== Private Interface =====----------------------// #ifdef USE_MPI MPI_Comm mCommunicator; +#else + MPI_Comm mCommunicator = 0; #endif // No public constructors, destructor, or assignment. diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 3ef35e9ad..c54a4e894 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -68,7 +68,7 @@ namespace Spheral { inline void spheral_adiak_init() { - adiak::init((void*) Communicator::communicator()); + adiak::init(Communicator::instance().comm_ptr()); // Always collect some curated default adiak information adiak::adiakversion(); adiak::user(); From d488e509299c44e255edfe600643a149924a4479 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 20 Aug 2024 17:12:16 -0700 Subject: [PATCH 09/37] Moved MPI finalize to occur when SpheralUtilities exits (either through Caliper or the communicator), allow caliper_config to be turned off at runtime if necessary, moved the instance of TimerMgr into the functions to simplify the calls --- src/Distributed/Communicator.hh | 2 + src/Distributed/mpi_mpi4py.py | 1 + src/PYB11/Utilities/Adiak.py | 7 +++- src/PYB11/Utilities/TimerMgr.py | 12 ++++++ src/PYB11/Utilities/Utilities_PYB11.py | 8 +++- src/SimulationControl/SpheralOptionParser.py | 13 +++++-- src/Utilities/Timer.hh | 41 +++++++++++--------- 7 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index 19437d56d..f4eeaeecc 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -27,8 +27,10 @@ public: static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } #ifdef USE_MPI static MPI_Comm* comm_ptr() { return &(instance().mCommunicator); } + static int finalize() { return MPI_Finalize(); } #else static MPI_Comm* comm_ptr() { return nullptr; } + static int finalize() { return 0; } #endif private: diff --git a/src/Distributed/mpi_mpi4py.py b/src/Distributed/mpi_mpi4py.py index e1692e256..5f99f1755 100644 --- a/src/Distributed/mpi_mpi4py.py +++ b/src/Distributed/mpi_mpi4py.py @@ -11,6 +11,7 @@ # as supported, but seem to be broken. import mpi4py mpi4py.rc.recv_mprobe = False +mpi4py.rc.finalize = False # Now go on as usual... from mpi4py import MPI diff --git a/src/PYB11/Utilities/Adiak.py b/src/PYB11/Utilities/Adiak.py index 06c1a9187..9bfd5cb05 100644 --- a/src/PYB11/Utilities/Adiak.py +++ b/src/PYB11/Utilities/Adiak.py @@ -6,9 +6,14 @@ # adiak::init() is called automatically when this module is loaded # adiak::fini() is called automatically when this module is destroyed +@PYB11cppname("adiak::fini") +def adiak_fini(): + "Finish Adiak" + return "void" + @PYB11cppname("adiak::collect_all") def adiak_collect_all(): - "Collect all default Adiak metadata" + "Add some default Adiak metadata" return "void" adiak_categories = PYB11enum(("unset", "all", "general", "performance", "control"), diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index 07d88e80e..52c4f7338 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -14,30 +14,42 @@ def instance(self): "Access the singleton instance of the timer manager" return "TimerMgr&" + @PYB11static def timer_start(self, region_name = "std::string"): "Start custom region Caliper timer, must have corresponding timer_end call" return "void" + @PYB11static def timer_end(self, region_name = "std::string"): "End custom region Caliper timer" return "void" + @PYB11static + def is_started(self): + "Check if ConfigManager has been started" + return "bool" + + @PYB11static def add(self, config_str = "std::string"): "Add a Caliper configuration" return "void" + @PYB11static def default_start(self, testname = "std::string"): "Set the spot Caliper configuration and start the manager" return "void" + @PYB11static def start(self): "Start the Caliper configuration manager" return "void" + @PYB11static def stop(self): "Stop the Caliper configuration manager" return "void" + @PYB11static def fini(self): "Flush the Caliper configuration manager" return "void" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index c54a4e894..479385596 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -68,7 +68,7 @@ namespace Spheral { inline void spheral_adiak_init() { - adiak::init(Communicator::instance().comm_ptr()); + adiak::init((void*) Communicator::comm_ptr()); // Always collect some curated default adiak information adiak::adiakversion(); adiak::user(); @@ -104,7 +104,11 @@ atexit.attr("register")(py::cpp_function([]() { TIME_END("main"); adiak::fini(); - Spheral::TimerMgr::instance().fini(); + if (Spheral::TimerMgr::is_started()) { + Spheral::TimerMgr::fini(); + } else { + int final = Communicator::finalize(); + } })); """ diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 008e80537..b4dd6b8c8 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -1,5 +1,5 @@ #------------------------------------------------------------------------------- -# Create a standard and hopefully convenient command line parser for Spheral +# Create a standard and hopefully convenient command line parser for Spheral # scripts. #------------------------------------------------------------------------------- import argparse @@ -46,13 +46,18 @@ def commandLine(**options): if (type(val) != type(options[key])): val = eval(val, gd) gd[key] = val + off_tests = ["none", "off", "disable", "disabled"] + if (not args.caliper_config.lower() in off_tests): + InitTimers(args) + return +def InitTimers(args): if (args.caliper_config): - TimerMgr.instance().add(args.caliper_config) - TimerMgr.instance().start() + TimerMgr.add(args.caliper_config) + TimerMgr.start() else: import random, os, sys unique_digits = ''.join(random.sample('0123456789', 4)) testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits - TimerMgr.instance().default_start(testname) + TimerMgr.default_start(testname) return diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 66d0426c5..fc64801fc 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -30,28 +30,32 @@ private: ~TimerMgr() { } TimerMgr(const TimerMgr&) = delete; TimerMgr& operator=(const TimerMgr&) = delete; + bool started = false; public: static TimerMgr& instance() { static TimerMgr theInstance; return theInstance; } - void timer_start(std::string regionName) { + static void timer_start(std::string regionName) { TIME_BEGIN(regionName.c_str()); } - void timer_end(std::string regionName) { + static void timer_end(std::string regionName) { TIME_END(regionName.c_str()); } + static bool is_started() { + return instance().started; + } #ifdef TIMER private: cali::ConfigManager cali_mgr; public: - void add(std::string config_str) { - bool test = cali_mgr.add(config_str.c_str()); - VERIFY2(test, cali_mgr.error_msg()); + static void add(std::string config_str) { + bool test = instance().cali_mgr.add(config_str.c_str()); + VERIFY2(test, instance().cali_mgr.error_msg()); } - void default_start(std::string testname) { + static void default_start(std::string testname) { if (!testname.empty()) { - std::string default_config = "spot,output=" + testname + ".cali,mem.highwatermark"; + std::string default_config = "spot,mem.highwatermark,output=" + testname + ".cali"; add(default_config); start(); } else if (Spheral::Process::getRank() == 0) { @@ -59,25 +63,26 @@ public: << "no Caliper configuration started" << std::endl; } } - void start() { - cali_mgr.start(); + static void start() { + instance().cali_mgr.start(); + instance().started = true; } - void stop() { - cali_mgr.stop(); + static void stop() { + instance().cali_mgr.stop(); } - void fini() { - cali_mgr.flush(); + static void fini() { + instance().cali_mgr.flush(); } #else - void default_start(std::string) { + static void default_start(std::string) { } - void add(std::string) { + static void add(std::string) { } - void start() { + static void start() { } - void stop() { + static void stop() { } - void fini() { + static void fini() { } #endif }; From 229459eb9a2615af508c6cd4cd0eacbd2f2e69f0 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 21 Aug 2024 08:52:30 -0700 Subject: [PATCH 10/37] Verify MPI finalize error inside communicator routine --- src/Distributed/Communicator.hh | 7 +++++-- src/PYB11/Utilities/Utilities_PYB11.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index f4eeaeecc..4dd326ab4 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -27,10 +27,13 @@ public: static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } #ifdef USE_MPI static MPI_Comm* comm_ptr() { return &(instance().mCommunicator); } - static int finalize() { return MPI_Finalize(); } + static void finalize() { + int finalize = MPI_Finalize(); + VERIFY(finalize); + } #else static MPI_Comm* comm_ptr() { return nullptr; } - static int finalize() { return 0; } + static void finalize() { return; } #endif private: diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 479385596..7a73d0619 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -107,7 +107,7 @@ if (Spheral::TimerMgr::is_started()) { Spheral::TimerMgr::fini(); } else { - int final = Communicator::finalize(); + Communicator::finalize(); } })); """ From 5a82358f0b696eb616e09b1a3aad5739cc93f066 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 21 Aug 2024 09:54:44 -0700 Subject: [PATCH 11/37] Moved ifdefs in Communicator.hh --- src/Distributed/Communicator.hh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index 4dd326ab4..56d5e3665 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -25,16 +25,19 @@ public: // Access the communicator. static MPI_Comm& communicator() { return instance().mCommunicator; } static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } + static MPI_Comm* comm_ptr() { #ifdef USE_MPI - static MPI_Comm* comm_ptr() { return &(instance().mCommunicator); } + return &(instance().mCommunicator); +#else + return nullptr; +#endif + } static void finalize() { +#ifdef USE_MPI int finalize = MPI_Finalize(); VERIFY(finalize); - } -#else - static MPI_Comm* comm_ptr() { return nullptr; } - static void finalize() { return; } #endif + } private: //------------------------===== Private Interface =====----------------------// From 03503b671b333276261c8d7a2653799f620105a6 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 21 Aug 2024 10:10:06 -0700 Subject: [PATCH 12/37] Forgot to include DBC.hh --- src/Distributed/Communicator.hh | 1 + src/SimulationControl/SpheralOptionParser.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index 56d5e3665..b59430953 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -9,6 +9,7 @@ #ifdef USE_MPI #include +#include "Utilities/DBC.hh" #else typedef int MPI_Comm; #endif diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index b4dd6b8c8..8947e3ccf 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -46,18 +46,22 @@ def commandLine(**options): if (type(val) != type(options[key])): val = eval(val, gd) gd[key] = val - off_tests = ["none", "off", "disable", "disabled"] - if (not args.caliper_config.lower() in off_tests): - InitTimers(args) + InitTimers(args.caliper_config) return -def InitTimers(args): - if (args.caliper_config): +def InitTimers(caliper_config, filename=""): + off_tests = ["none", "off", "disable", "disabled"] + if (caliper_config.lower() in off_tests): + return + elif (caliper_config): TimerMgr.add(args.caliper_config) TimerMgr.start() else: import random, os, sys - unique_digits = ''.join(random.sample('0123456789', 4)) - testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits + if (filename): + testname = filename + else: + unique_digits = ''.join(random.sample('0123456789', 4)) + testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits TimerMgr.default_start(testname) return From da9539e333ecd01a740cebfb4dc22295243d7b57 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 21 Aug 2024 16:04:38 -0700 Subject: [PATCH 13/37] Added more checks for mpi finalize, moved communicator routines to cc file --- src/Distributed/Communicator.cc | 32 +++++++++++++++++++++++++++++--- src/Distributed/Communicator.hh | 19 ++----------------- src/Utilities/Timer.hh | 1 + 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/Distributed/Communicator.cc b/src/Distributed/Communicator.cc index 96aa5a236..b913cb4bc 100644 --- a/src/Distributed/Communicator.cc +++ b/src/Distributed/Communicator.cc @@ -13,12 +13,11 @@ namespace Spheral { // Default constructor (private). //------------------------------------------------------------------------------ Communicator:: -Communicator() +Communicator() { #ifdef USE_MPI - : mCommunicator() { mCommunicator = MPI_COMM_WORLD; #else -{ + mCommunicator = 0; #endif } @@ -29,4 +28,31 @@ Communicator:: ~Communicator() { } +//------------------------------------------------------------------------------ +// Public routines +//------------------------------------------------------------------------------ + +MPI_Comm* Communicator::comm_ptr() { +#ifdef USE_MPI + return &(instance().mCommunicator); +#else + return nullptr; +#endif +} + +void Communicator::finalize() { +#ifdef USE_MPI + int finalized = 0; + MPI_Finalized(&finalized); + if (finalized != 0) { + int finalize = MPI_Finalize(); + if (finalize != 0) { + char* string = nullptr; + int resultlen = 0; + MPI_Error_string(finalize, string, &resultlen); + VERIFY2(finalize, string); + } + } +#endif +} } diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index b59430953..f9c2a6808 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -26,27 +26,12 @@ public: // Access the communicator. static MPI_Comm& communicator() { return instance().mCommunicator; } static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } - static MPI_Comm* comm_ptr() { -#ifdef USE_MPI - return &(instance().mCommunicator); -#else - return nullptr; -#endif - } - static void finalize() { -#ifdef USE_MPI - int finalize = MPI_Finalize(); - VERIFY(finalize); -#endif - } + static MPI_Comm* comm_ptr(); + static void finalize(); private: //------------------------===== Private Interface =====----------------------// -#ifdef USE_MPI MPI_Comm mCommunicator; -#else - MPI_Comm mCommunicator = 0; -#endif // No public constructors, destructor, or assignment. Communicator(); diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index fc64801fc..e770a2988 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -72,6 +72,7 @@ public: } static void fini() { instance().cali_mgr.flush(); + instance().started = false; } #else static void default_start(std::string) { From 92e9706c796c3626fb0c5cbce95ce43d46cb2f81 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 22 Aug 2024 09:51:21 -0700 Subject: [PATCH 14/37] Added timer test script and changed CMake logic for making tests, added config and filename caliper variables --- cmake/SetupSpheral.cmake | 2 +- src/PYB11/Utilities/TimerMgr.py | 10 +++ src/PYB11/Utilities/Utilities_PYB11.py | 2 +- src/SimulationControl/SpheralOptionParser.py | 16 ++-- src/Utilities/Timer.hh | 10 +++ tests/integration.ats | 1 + tests/unit/CMakeLists.txt | 11 +++ tests/unit/Utilities/testTimers.py.in | 78 ++++++++++++++++++++ 8 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 tests/unit/CMakeLists.txt create mode 100644 tests/unit/Utilities/testTimers.py.in diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index b13c72e51..e7c77fa2c 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -152,7 +152,7 @@ endif() # Build C++ tests and install tests to install directory #------------------------------------------------------------------------------- if (ENABLE_TESTS) - add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit/CXXTests) + add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit) # A macro to preserve directory structure when installing files macro(install_with_directory) diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index 52c4f7338..7657bc6ca 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -53,3 +53,13 @@ def stop(self): def fini(self): "Flush the Caliper configuration manager" return "void" + + @PYB11static + def get_config(self): + "Return the current Caliper configuration" + return "std::string" + + @PYB11static + def get_filename(self): + "Return current Caliper filename, if set" + return "std::string" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 7a73d0619..feba85364 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -99,7 +99,7 @@ TIME_BEGIN("main"); Spheral::spheral_adiak_init(); -// Call these routines when module is destroyed +// Call these routines when module is exited auto atexit = py::module_::import("atexit"); atexit.attr("register")(py::cpp_function([]() { TIME_END("main"); diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 8947e3ccf..f5168e734 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -24,7 +24,8 @@ def commandLine(**options): dest = "verbose", default = False, help = "Verbose output -- print all options that were set.") - parser.add_argument("--caliper-config", default="", type=str) + parser.add_argument("--caliperConfig", default="", type=str) + parser.add_argument("--caliperFilename", default="", type=str) # Evaluate the command line. args = parser.parse_args() arg_dict = vars(args) @@ -38,23 +39,26 @@ def commandLine(**options): print(" * ", key, " = ", val) else: print(" ", key, " = ", val) - + if (args.caliperConfig): + print(" * caliperConfig = ", args.caliperConfig) + if (args.caliperFilename): + print(" * caliperFilename = ", args.caliperFilename) # Set all the variables. gd = globalFrame().f_globals for key, val in arg_dict.items(): if key in options: if (type(val) != type(options[key])): val = eval(val, gd) - gd[key] = val - InitTimers(args.caliper_config) + gd[key] = val + InitTimers(args.caliperConfig, args.caliperFilename) return -def InitTimers(caliper_config, filename=""): +def InitTimers(caliper_config, filename): off_tests = ["none", "off", "disable", "disabled"] if (caliper_config.lower() in off_tests): return elif (caliper_config): - TimerMgr.add(args.caliper_config) + TimerMgr.add(caliper_config) TimerMgr.start() else: import random, os, sys diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index e770a2988..84838730d 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -31,6 +31,8 @@ private: TimerMgr(const TimerMgr&) = delete; TimerMgr& operator=(const TimerMgr&) = delete; bool started = false; + std::string caliperFilename = ""; + std::string caliperConfig = ""; public: static TimerMgr& instance() { static TimerMgr theInstance; @@ -45,6 +47,12 @@ public: static bool is_started() { return instance().started; } + static std::string get_config() { + return instance().caliperConfig; + } + static std::string get_filename() { + return instance().caliperFilename; + } #ifdef TIMER private: cali::ConfigManager cali_mgr; @@ -52,10 +60,12 @@ public: static void add(std::string config_str) { bool test = instance().cali_mgr.add(config_str.c_str()); VERIFY2(test, instance().cali_mgr.error_msg()); + instance().caliperConfig += config_str; } static void default_start(std::string testname) { if (!testname.empty()) { std::string default_config = "spot,mem.highwatermark,output=" + testname + ".cali"; + instance().caliperFilename = testname + ".cali"; add(default_config); start(); } else if (Spheral::Process::getRank() == 0) { diff --git a/tests/integration.ats b/tests/integration.ats index 43863d2f7..2763d3e75 100644 --- a/tests/integration.ats +++ b/tests/integration.ats @@ -70,6 +70,7 @@ source("unit/Utilities/testCubicHermiteInterpolator.py") source("unit/Utilities/testBiLinearInterpolator.py") source("unit/Utilities/testBiQuadraticInterpolator.py") source("unit/Utilities/testBiCubicInterpolator.py") +source("unit/Utilities/testTimers.py") # Mesh tests. source("unit/Mesh/testLineMesh.py") diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt new file mode 100644 index 000000000..85ece892a --- /dev/null +++ b/tests/unit/CMakeLists.txt @@ -0,0 +1,11 @@ +add_subdirectory(CXXTests) + +if (ENABLE_TIMER) + set(MPI_TIMER_VAR "True") +else() + set(MPI_TIMER_VAR "False") +endif() +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/Utilities/testTimers.py.in" + "${SPHERAL_TEST_INSTALL_PREFIX}/tests/unit/Utilities/testTimers.py" + ) diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in new file mode 100644 index 000000000..612118346 --- /dev/null +++ b/tests/unit/Utilities/testTimers.py.in @@ -0,0 +1,78 @@ +# +# +#ATS:tt0 = test(SELF, "--caliperFilename 'timer_test_1'", label="Timer test 1", np=8) +#ATS:tt1 = test(SELF, "--caliperConfig 'None' --caliperFilename 'timer_test_2'", label="Timer test 2", np=8) +#ATS:tt2 = test(SELF, "--caliperFilename 'timer_test_3'", label="Timer test 3", np=1) +# + +import Spheral +from SpheralTestUtilities import * +from SpheralOptionParser import * +from SpheralUtilities import TimerMgr +from SpheralUtilities import * +import mpi + +import sys, os, time +caliper_loc = "@caliper_DIR@" +sys.path.append(os.path.join(caliper_loc, "../../../lib64/caliper")) + +import caliperreader as cr + +timer_compiler = @MPI_TIMER_VAR@ + +timer_test_name = "timer_test" +commandLine() + +# Remove cali files from previous test runs +caliper_file = TimerMgr.get_filename() +if (os.path.exists(caliper_file)): + if (mpi.rank == 0): + os.remove(caliper_file) + +do_timers = False +if (TimerMgr.is_started()): + do_timers = True +test_dict_0 = {"perf_test": "weak_scaling"} +adiak_valueString("perf_test", test_dict_0["perf_test"], adiak_categories.performance) +test_dict_1 = {"rank_count": mpi.procs} +adiak_valueInt("rank_count", test_dict_1["rank_count"]) +test_dicts = [test_dict_0, test_dict_1] +run_count = 8 +sleep_time = 1.E-4 +fake_timer_name = "test_timer" + +for i in range(run_count): + TimerMgr.timer_start(fake_timer_name) + time.sleep(sleep_time) + TimerMgr.timer_end(fake_timer_name) +if (do_timers and TimerMgr.get_filename()): + adiak_fini() + TimerMgr.fini() + mpi.barrier() + if (not os.path.exists(caliper_file)): + raise ValueError("Caliper file not created") + r = cr.CaliperReader() + r.read(caliper_file) + records = r.records + found_errors = 0 + # Test for timer name + if (fake_timer_name in records[1]['region']): + print(f"Found {fake_timer_name} timer") + else: + found_errors += 1 + # Test for function count + count_val = int(eval(records[1]["avg#sum#rc.count"])) + if (count_val == run_count): + print("Run count in Caliper file is correct") + else: + found_errors += 1 + # Test for adiak values + for td in test_dicts: + if (td.items() <= r.globals.items()): + print(f"Found {td.items()}") + else: + found_errors += 1 + if (found_errors > 0): + raise ValueError("Caliper file not correct") + else: + print("No errors found for TimerMgr") From 5b46839cfed96eb881c1ecfc02a83d71312ef29e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 22 Aug 2024 10:48:02 -0700 Subject: [PATCH 15/37] Added warning for caliper configurations provided to incorrect builds, added documentation, updated copyright for docs to 2024 --- docs/conf.py | 2 +- docs/conf.py.in | 2 +- docs/developer/dev/diagnostic_tools.rst | 49 +++++++++++++++++--- docs/intro/introduction.rst.inc | 2 +- src/PYB11/Utilities/TimerMgr.py | 5 ++ src/SimulationControl/SpheralOptionParser.py | 11 ++++- src/Utilities/Timer.hh | 6 +++ tests/unit/Utilities/testTimers.py.in | 9 ++-- 8 files changed, 71 insertions(+), 15 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index df9b1060e..a8619454d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ # -- Project information ----------------------------------------------------- project = 'Spheral' -copyright = '2012, LLNS' +copyright = '2024, LLNS' author = 'J. Michael Owen' # The short X.Y version diff --git a/docs/conf.py.in b/docs/conf.py.in index b7d23f29a..2f7bd11ad 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -22,7 +22,7 @@ import sphinx_rtd_theme # -- Project information ----------------------------------------------------- project = 'Spheral' -copyright = '2012, LLNS' +copyright = '2024, LLNS' author = 'J. Michael Owen' # The short X.Y version diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 07d4d2265..aaa1a2f2f 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -1,24 +1,46 @@ Diagnostics ########### -Spheral uses Caliper to preform diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ENABLE_TIMER=On. Otherwise the diagnostic regions are no-ops for improved preformance. +Spheral uses Caliper to preform diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ``ENABLE_TIMER=ON``. Otherwise the diagnostic regions are no-ops for improved preformance. :: - ./scripts/devtools/host-config-build.py -.cmake -DENABLE_TIMER=On + ./scripts/devtools/host-config-build.py -.cmake -DENABLE_TIMER=ON Querying using Caliper ====================== -By defualt, even when configured with ENABLE_TIMER=On, there is no information being recorded. Caliper uses command line options to report data, the simplest of which is ``CALI_CONFIG=runtime-report`` which reports the timing for the regions and prints them out in the terminal. For example: +Caliper is configured and started through the ``cali::ConfigManager``. +The ``cali::ConfigManager`` is wrapped in a ``TimerMgr`` singleton class, which has a python interface. +``TimerMgr`` is initialized and started in the ``InitTimers`` routine which is called in ``commandLine()`` in ``src/SimulationControl/SpheralOptionParser.py``. +By default, the Caliper configuration is set to ``spot,mem.highwatermark`` and output Caliper files. +The Caliper files are named based on what file is being run, for example: :: - CALI_CONFIG=runtime-report python Noh-cylindrical-2d.py + python Noh-cylindrical-2d.py + +will produce timing files called +:: + + Noh-cylindrical-2d_####.cali + +where the number signs are randomly generated numbers. +The Caliper file names for the default configuration can be overwritten using the command line +:: + + python Noh-cylindrical-2d.py --caliperFilename 'new_test_name' + +Non-default Caliper configurations can be set at the command line using ``--caliperConfig`` like so +:: + + python Noh-cylindrical-2d.py --caliperConfig 'runtime-report(output=time.txt),calc.inclusive,region.count' + +Additionally, Caliper timers can be turned off using ``--caliperConfig none``. .. note:: To obtain a similar result to that of the removed Spheral::Timer use :kbd:`CALI_CONFIG=runtime\-report(output=time.txt),calc.inclusive,region.count` this will result in a file named time.txt with cumulative times for the nested regions as well as a count of how many times each region ran. -There are many different options that can be used with ``CALI_CONFIG`` to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral in the future: +There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral in the future: * `Configuration basics `_ * `Builtin Configuration `_ @@ -26,8 +48,8 @@ There are many different options that can be used with ``CALI_CONFIG`` to view v * `Output Format `_ -Adding Regions -============== +Adding Region Timers in C++ +=========================== So far there are two different types of regions in Spheral, using the following macros: :: @@ -39,3 +61,16 @@ So far there are two different types of regions in Spheral, using the following - ``TIME_FUNCTION`` can be added to the very beginning of a function and creates a region for the entire function using the function's name. ``TIME_FUNCTION`` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. - ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case timer_name) as the name. + + +Adding Region Timers in Python +============================== + +Region timers can be added inside the python code using the following function calls: +:: + + TimerMgr.timer_start("some_function") + some_function_call() + TimerMgr.timer_end("some_function") + +It is important that all timers have both a start and end call. Otherwise, memory issues will occur. diff --git a/docs/intro/introduction.rst.inc b/docs/intro/introduction.rst.inc index 1de8df747..47211f5d1 100644 --- a/docs/intro/introduction.rst.inc +++ b/docs/intro/introduction.rst.inc @@ -25,7 +25,7 @@ Some useful features are Release and License: #################### -Copyright (c) 2012, Lawrence Livermore National Security, LLC. +Copyright (c) 2024, Lawrence Livermore National Security, LLC. Produced at the Lawrence Livermore National Laboratory Written by J. Michael Owen mikeowen@llnl.gov LLNL-CODE-561852 diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index 7657bc6ca..4f61a83aa 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -63,3 +63,8 @@ def get_config(self): def get_filename(self): "Return current Caliper filename, if set" return "std::string" + + @PYB11static + def timers_usable(self): + "Return whether the code has been compiled with timers turned on" + return "bool" diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index f5168e734..16bc48595 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -30,6 +30,12 @@ def commandLine(**options): args = parser.parse_args() arg_dict = vars(args) + if (not TimerMgr.timers_usable()): + if (args.caliperConfig or args.caliperFilename): + print("WARNING: Caliper command line inputs provided for "+\ + "non-timer install. Reconfigure the install with "+\ + "-DENABLE_TIMER=ON to be able to use Caliper timers.") + # Verbose output? if args.verbose: print("All parameters set:") @@ -54,7 +60,7 @@ def commandLine(**options): return def InitTimers(caliper_config, filename): - off_tests = ["none", "off", "disable", "disabled"] + off_tests = ["none", "off", "disable", "disabled", "0"] if (caliper_config.lower() in off_tests): return elif (caliper_config): @@ -64,6 +70,9 @@ def InitTimers(caliper_config, filename): import random, os, sys if (filename): testname = filename + # Remove the cali file extension + if (".cali" in testname): + testname = testname.replace(".cali", "") else: unique_digits = ''.join(random.sample('0123456789', 4)) testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 84838730d..14154411c 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -57,6 +57,9 @@ public: private: cali::ConfigManager cali_mgr; public: + static bool timers_usable() { + return true; + } static void add(std::string config_str) { bool test = instance().cali_mgr.add(config_str.c_str()); VERIFY2(test, instance().cali_mgr.error_msg()); @@ -85,6 +88,9 @@ public: instance().started = false; } #else + static bool timers_usable() { + return false; + } static void default_start(std::string) { } static void add(std::string) { diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in index 612118346..71c51e234 100644 --- a/tests/unit/Utilities/testTimers.py.in +++ b/tests/unit/Utilities/testTimers.py.in @@ -1,8 +1,8 @@ # # -#ATS:tt0 = test(SELF, "--caliperFilename 'timer_test_1'", label="Timer test 1", np=8) -#ATS:tt1 = test(SELF, "--caliperConfig 'None' --caliperFilename 'timer_test_2'", label="Timer test 2", np=8) -#ATS:tt2 = test(SELF, "--caliperFilename 'timer_test_3'", label="Timer test 3", np=1) +#ATS:test(SELF, "--caliperFilename 'timer_test_1'", label="Timer test 1", np=8) +#ATS:test(SELF, "--caliperConfig 'None'", label="Timer test 2", np=8) +#ATS:test(SELF, "--caliperFilename 'timer_test_3'", label="Timer test 3", np=1) # import Spheral @@ -18,6 +18,7 @@ sys.path.append(os.path.join(caliper_loc, "../../../lib64/caliper")) import caliperreader as cr +# Set based on ENABLE_TIMERS configure variable timer_compiler = @MPI_TIMER_VAR@ timer_test_name = "timer_test" @@ -50,7 +51,7 @@ if (do_timers and TimerMgr.get_filename()): TimerMgr.fini() mpi.barrier() if (not os.path.exists(caliper_file)): - raise ValueError("Caliper file not created") + raise ValueError("Caliper file not found") r = cr.CaliperReader() r.read(caliper_file) records = r.records From c010fb7c823202568e6565077b488e5adc4f8d77 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 22 Aug 2024 11:19:02 -0700 Subject: [PATCH 16/37] Fix bug with getting caliperreader from TPL --- cmake/InstallTPLs.cmake | 2 ++ scripts/performance/performance.py.in | 4 ++-- tests/unit/Utilities/testTimers.py.in | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index ba8a149c3..fe12b1fde 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -92,6 +92,8 @@ endif() message("-----------------------------------------------------------------------------") # Use find_package to get caliper if (ENABLE_TIMER) + # Save caliper_DIR because it gets overwritten by find_package + set(CONFIG_CALIPER_DIR "${caliper_DIR}" CACHE PATH "Configuration Caliper directory") find_package(caliper REQUIRED NO_DEFAULT_PATH PATHS ${caliper_DIR}/share/cmake/caliper) if(caliper_FOUND) list(APPEND SPHERAL_BLT_DEPENDS caliper) diff --git a/scripts/performance/performance.py.in b/scripts/performance/performance.py.in index 6459fd4d5..b337e5042 100644 --- a/scripts/performance/performance.py.in +++ b/scripts/performance/performance.py.in @@ -1,8 +1,8 @@ #!/user/bin/env python3 import sys, os -caliper_loc = "@caliper_DIR@" -sys.path.append(os.path.join(caliper_loc, "../../../lib64/caliper")) +caliper_loc = "@CONFIG_CALIPER_DIR@" +sys.path.append(os.path.join(caliper_loc, "lib64/caliper")) import caliperreader as cr diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in index 71c51e234..b34cb00f1 100644 --- a/tests/unit/Utilities/testTimers.py.in +++ b/tests/unit/Utilities/testTimers.py.in @@ -13,10 +13,7 @@ from SpheralUtilities import * import mpi import sys, os, time -caliper_loc = "@caliper_DIR@" -sys.path.append(os.path.join(caliper_loc, "../../../lib64/caliper")) -import caliperreader as cr # Set based on ENABLE_TIMERS configure variable timer_compiler = @MPI_TIMER_VAR@ @@ -50,6 +47,9 @@ if (do_timers and TimerMgr.get_filename()): adiak_fini() TimerMgr.fini() mpi.barrier() + caliper_loc = "@CONFIG_CALIPER_DIR@" + sys.path.append(os.path.join(caliper_loc, "lib64/caliper")) + import caliperreader as cr if (not os.path.exists(caliper_file)): raise ValueError("Caliper file not found") r = cr.CaliperReader() From b4a6e92b2b8893ac7fbc24c22f12cf648598c1cd Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 22 Aug 2024 15:37:58 -0700 Subject: [PATCH 17/37] Fixed timer test to accomodate issue with caliperreader --- tests/unit/Utilities/testTimers.py.in | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in index b34cb00f1..85902e6c3 100644 --- a/tests/unit/Utilities/testTimers.py.in +++ b/tests/unit/Utilities/testTimers.py.in @@ -18,7 +18,6 @@ import sys, os, time # Set based on ENABLE_TIMERS configure variable timer_compiler = @MPI_TIMER_VAR@ -timer_test_name = "timer_test" commandLine() # Remove cali files from previous test runs @@ -31,9 +30,19 @@ do_timers = False if (TimerMgr.is_started()): do_timers = True test_dict_0 = {"perf_test": "weak_scaling"} -adiak_valueString("perf_test", test_dict_0["perf_test"], adiak_categories.performance) -test_dict_1 = {"rank_count": mpi.procs} -adiak_valueInt("rank_count", test_dict_1["rank_count"]) +adiak_valueString("perf_test", test_dict_0["perf_test"], + adiak_categories.performance) +# Caliperreader reads everything as strings for some terrible reason +# So the test have to be hacked up + +# Correct method: +# test_dict_1 = {"rank_count": mpi.procs} +# adiak_valueInt("rank_count", test_dict_1["rank_count"]) + +# Hacked method to have tests pass with caliperreader: +test_dict_1 = {"rank_count": str(mpi.procs)} +adiak_valueString("rank_count", test_dict_1["rank_count"]) + test_dicts = [test_dict_0, test_dict_1] run_count = 8 sleep_time = 1.E-4 From cf674cd652b0cc426267cb54f5ecbe434434b542 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 4 Sep 2024 09:50:20 -0700 Subject: [PATCH 18/37] Added more default adiak values --- src/SimulationControl/SpheralOptionParser.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 16bc48595..107132a02 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -2,7 +2,7 @@ # Create a standard and hopefully convenient command line parser for Spheral # scripts. #------------------------------------------------------------------------------- -import argparse +import argparse, mpi from SpheralCompiledPackages import * @@ -56,6 +56,7 @@ def commandLine(**options): if (type(val) != type(options[key])): val = eval(val, gd) gd[key] = val + # Initialize timers InitTimers(args.caliperConfig, args.caliperFilename) return @@ -74,7 +75,15 @@ def InitTimers(caliper_config, filename): if (".cali" in testname): testname = testname.replace(".cali", "") else: + # Name file based on name of python file being run + # with four random digits at the end unique_digits = ''.join(random.sample('0123456789', 4)) - testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + "_" + unique_digits + # Remove any file extension provided + testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + # Remove any dashes + testname = testname.replace("-", "") + testname += unique_digits TimerMgr.default_start(testname) + adiak_valueInt("threads_per_rank", omp_get_num_threads()) + adiak_valueInt("num_ranks", mpi.procs) return From 5a68c4f3fa07adf67207c21b1b36ace3cd100080 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 5 Sep 2024 09:58:40 -0700 Subject: [PATCH 19/37] Fix random visit bug --- src/SimulationControl/SpheralVisitDump.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/SimulationControl/SpheralVisitDump.py b/src/SimulationControl/SpheralVisitDump.py index 9e772feed..16751280b 100644 --- a/src/SimulationControl/SpheralVisitDump.py +++ b/src/SimulationControl/SpheralVisitDump.py @@ -365,8 +365,7 @@ def _dumpField(self, field, file): # Private method to process a Field name into a Visit palatable label #--------------------------------------------------------------------------- def _processName(self, name): - import string - return string.replace(str(name), " ", "_") + return name.replace(" ", "_") #--------------------------------------------------------------------------- # Private method to write an atomic DataType element to a file. From f024462856c65527f3e1c50cc5ea9f848aea2faa Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 5 Sep 2024 11:12:29 -0700 Subject: [PATCH 20/37] Changed random digits for caliper file names to be the date and time, updated the documentation accordingly --- docs/developer/dev/diagnostic_tools.rst | 31 ++++++++++---------- src/SimulationControl/SpheralOptionParser.py | 5 ++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index aaa1a2f2f..9e166451e 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -10,11 +10,11 @@ Spheral uses Caliper to preform diagnostics, such as timing. To enable this func Querying using Caliper ====================== -Caliper is configured and started through the ``cali::ConfigManager``. -The ``cali::ConfigManager`` is wrapped in a ``TimerMgr`` singleton class, which has a python interface. -``TimerMgr`` is initialized and started in the ``InitTimers`` routine which is called in ``commandLine()`` in ``src/SimulationControl/SpheralOptionParser.py``. -By default, the Caliper configuration is set to ``spot,mem.highwatermark`` and output Caliper files. -The Caliper files are named based on what file is being run, for example: +Caliper is configured and started through the `cali::ConfigManager`. +The `cali::ConfigManager` is wrapped in a `TimerMgr` singleton class, which has a python interface. +`TimerMgr` is initialized and started in the `InitTimers` routine which is called in `commandLine()` in ``src/SimulationControl/SpheralOptionParser.py``. +By default, the Caliper configuration is set to ``spot,mem.highwatermark`` and outputs Caliper files (``.cali``). +For the default configuration, the Caliper files are named based on what file is being run, for example: :: python Noh-cylindrical-2d.py @@ -22,10 +22,10 @@ The Caliper files are named based on what file is being run, for example: will produce timing files called :: - Noh-cylindrical-2d_####.cali + Noh-cylindrical-2d_YEAR_MONTH_DATE_TIME.cali -where the number signs are randomly generated numbers. -The Caliper file names for the default configuration can be overwritten using the command line +where the file name includes the current date and time. +The Caliper file name can be specified using the command line :: python Noh-cylindrical-2d.py --caliperFilename 'new_test_name' @@ -35,12 +35,12 @@ Non-default Caliper configurations can be set at the command line using ``--cali python Noh-cylindrical-2d.py --caliperConfig 'runtime-report(output=time.txt),calc.inclusive,region.count' -Additionally, Caliper timers can be turned off using ``--caliperConfig none``. - .. note:: - To obtain a similar result to that of the removed Spheral::Timer use :kbd:`CALI_CONFIG=runtime\-report(output=time.txt),calc.inclusive,region.count` this will result in a file named time.txt with cumulative times for the nested regions as well as a count of how many times each region ran. + The above configuration produces timing results similar to the previous `Spheral::Timer` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. -There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral in the future: +Additionally, Caliper timers can be turned off using ``--caliperConfig none``. + +There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral: * `Configuration basics `_ * `Builtin Configuration `_ @@ -58,9 +58,9 @@ So far there are two different types of regions in Spheral, using the following TIME_BEGIN("timer_name") TIME_END("timer_name") -- ``TIME_FUNCTION`` can be added to the very beginning of a function and creates a region for the entire function using the function's name. ``TIME_FUNCTION`` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. +- `TIME_FUNCTION` can be added to the very beginning of a function and creates a region for the entire function using the function's name. `TIME_FUNCTION` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. -- ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case timer_name) as the name. +- `TIME_BEGIN("timer_name")` and `TIME_END("timer_name")` create a region between the two different calls and use the string (in this case timer_name) as the name. Adding Region Timers in Python @@ -73,4 +73,5 @@ Region timers can be added inside the python code using the following function c some_function_call() TimerMgr.timer_end("some_function") -It is important that all timers have both a start and end call. Otherwise, memory issues will occur. +.. note:: + IMPORTANT: All timers must have both a start and end call. Otherwise, memory issues will occur. diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 107132a02..3ff5f9b0e 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -75,9 +75,10 @@ def InitTimers(caliper_config, filename): if (".cali" in testname): testname = testname.replace(".cali", "") else: + from datetime import datetime + # Append the current day and time to the filename + unique_digits = datetime.now().strftime("_%Y_%m_%d_%H%M%S") # Name file based on name of python file being run - # with four random digits at the end - unique_digits = ''.join(random.sample('0123456789', 4)) # Remove any file extension provided testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] # Remove any dashes From 4c21cbdc0954b13648b20171d67f116631d52614 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 5 Sep 2024 11:17:34 -0700 Subject: [PATCH 21/37] Prevent automatically adding file extensions to caliper file names --- docs/developer/dev/diagnostic_tools.rst | 2 +- src/SimulationControl/SpheralOptionParser.py | 8 +------- src/Utilities/Timer.hh | 4 ++-- tests/unit/Utilities/testTimers.py.in | 4 ++-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 9e166451e..429e8e6a9 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -28,7 +28,7 @@ where the file name includes the current date and time. The Caliper file name can be specified using the command line :: - python Noh-cylindrical-2d.py --caliperFilename 'new_test_name' + python Noh-cylindrical-2d.py --caliperFilename 'new_test_name.cali' Non-default Caliper configurations can be set at the command line using ``--caliperConfig`` like so :: diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 3ff5f9b0e..9955a5299 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -71,19 +71,13 @@ def InitTimers(caliper_config, filename): import random, os, sys if (filename): testname = filename - # Remove the cali file extension - if (".cali" in testname): - testname = testname.replace(".cali", "") else: from datetime import datetime # Append the current day and time to the filename unique_digits = datetime.now().strftime("_%Y_%m_%d_%H%M%S") # Name file based on name of python file being run - # Remove any file extension provided testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] - # Remove any dashes - testname = testname.replace("-", "") - testname += unique_digits + testname += unique_digits + ".cali" TimerMgr.default_start(testname) adiak_valueInt("threads_per_rank", omp_get_num_threads()) adiak_valueInt("num_ranks", mpi.procs) diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 14154411c..1188ce79b 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -67,8 +67,8 @@ public: } static void default_start(std::string testname) { if (!testname.empty()) { - std::string default_config = "spot,mem.highwatermark,output=" + testname + ".cali"; - instance().caliperFilename = testname + ".cali"; + std::string default_config = "spot,mem.highwatermark,output=" + testname; + instance().caliperFilename = testname; add(default_config); start(); } else if (Spheral::Process::getRank() == 0) { diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in index 85902e6c3..dea75e2f9 100644 --- a/tests/unit/Utilities/testTimers.py.in +++ b/tests/unit/Utilities/testTimers.py.in @@ -1,8 +1,8 @@ # # -#ATS:test(SELF, "--caliperFilename 'timer_test_1'", label="Timer test 1", np=8) +#ATS:test(SELF, "--caliperFilename 'timer_test_1.cali'", label="Timer test 1", np=8) #ATS:test(SELF, "--caliperConfig 'None'", label="Timer test 2", np=8) -#ATS:test(SELF, "--caliperFilename 'timer_test_3'", label="Timer test 3", np=1) +#ATS:test(SELF, "--caliperFilename 'timer_test_3.cali'", label="Timer test 3", np=1) # import Spheral From 209ffa597f0a3f20f88674737942c24deb13d429 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 5 Sep 2024 11:25:29 -0700 Subject: [PATCH 22/37] Added caliper and adiak to TPL list, changed italics to bold for default options since italics was too subtle --- .../include/appendecies/cmake_config.rst.inc | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/build_guide/include/appendecies/cmake_config.rst.inc b/docs/build_guide/include/appendecies/cmake_config.rst.inc index bcf0e8f60..4fb343fa6 100644 --- a/docs/build_guide/include/appendecies/cmake_config.rst.inc +++ b/docs/build_guide/include/appendecies/cmake_config.rst.inc @@ -28,6 +28,8 @@ For just the C++ compiled Spheral a number of TPLs are required: - Polyclipper - Conduit - Axom +- Adiak +- Caliper There are also a number of libraries / python packages that are required for compiling the python bindings and executing Spheral at runtime: @@ -66,58 +68,58 @@ OpenMP and MPI support is handled through BLT. Use the option flags ``-DENABLE_ CMake variables -------------------- -In this section we list the CMake variables that can be tweaked for a Spheral build. Where appropriate the options are listed, with the default value in *italics*. +In this section we list the CMake variables that can be tweaked for a Spheral build. Where appropriate the options are listed, with the default value in **bold**. -``CMAKE_BUILD_TYPE`` (Debug, *Release*, RelWithDebInfo, MinSizeRel) +``CMAKE_BUILD_TYPE`` (Debug, **Release**, RelWithDebInfo, MinSizeRel) Choose the type of build -- for more information see the `CMake documentation `_. ``CMAKE_INSTALL_PREFIX`` The top-level path for installing Spheral include files, libraries, and any Python modules or documentation. -``ENABLE_CXXONLY`` (On, *Off*) +``ENABLE_CXXONLY`` (On, **Off**) Do not build python wrappers for Spheral. -``ENABLE_STATIC_CXXONLY`` (On, *Off*) +``ENABLE_STATIC_CXXONLY`` (On, **Off**) Do not build python wrappers for Spheral. Build and export static library files for Spheral. -``ENABLE_SHARED`` (*On*, Off) +``ENABLE_SHARED`` (**On**, Off) Build Spheral C++ libraries as shared libraries. -``ENABLE_DEV_BUILD`` (On, *Off*) +``ENABLE_DEV_BUILD`` (On, **Off**) Builds separate internal C++ libraries for faster code development. ``_DIR`` Directory of previously built TPL. -``ENABLE_STATIC_TPL`` (On, *Off*) +``ENABLE_STATIC_TPL`` (On, **Off**) Link static libraries instead of shared for HDF5 and Conduit. -``ENABLE_OPENMP`` (*On*, Off) +``ENABLE_OPENMP`` (**On**, Off) Support for OpenMP. -``ENABLE_MPI`` (*On*, Off) +``ENABLE_MPI`` (**On**, Off) Support for MPI. -``ENABLE_1D`` (*On*, Off) +``ENABLE_1D`` (**On**, Off) Build Spheral with 1D support. -``ENABLE_2D`` (*On*, Off) +``ENABLE_2D`` (**On**, Off) Build Spheral with 2D support. -``ENABLE_3D`` (*On*, Off) +``ENABLE_3D`` (**On**, Off) Build Spheral with 3D support. -``ENABLE_ANEOS`` (*On*, Off) +``ENABLE_ANEOS`` (**On**, Off) Install the ANEOS (Analytics Equation of State) package along with the Spheral interface to it. This is a legacy equation of state frequently used for geophysical materials. See descriptions in the `iSALE `_ documentation. -``ENABLE_HELMHOLTZ`` (*On*, Off) +``ENABLE_HELMHOLTZ`` (**On**, Off) Compile the included Helmholtz equation of state, typically used in astrophysical calculations. See a discussion `here `_. -``ENABLE_OPENSUBDIV`` (*On*, Off) +``ENABLE_OPENSUBDIV`` (**On**, Off) Install the Opensubdiv library along with the Spheral interface to it. Opensubdiv is a `Pixar provided library `_, which Spheral uses to implement refinement of polyhedra for some specialized problem generation capabilities. -``ENABLE_TIMER`` (*On*, Off) - Enable timer information from Spheral. +``ENABLE_TIMER`` (On, **Off**) + Enable Caliper timer information for Spheral. ``DBC_MODE`` (None, All, Pre) Set the compile time design by contract (DBC) mode for Spheral. Design by contract statements are very useful developer tools, whereby the developer can insert tests in the code as they write it. These statements are both useful for tracking down bugs with fine-grained testing throughout the code, as well as useful documentation in the code about what sort of conditions are expected to hold. @@ -138,23 +140,23 @@ In this section we list the CMake variables that can be tweaked for a Spheral bu It is worth noting ``DBC_MODE=All`` is quite expensive at run time (of order 4x more), so this is not intended to be active for a release/production compilation of Spheral. -``ENABLE_WARNINGS`` (On, *Off*) +``ENABLE_WARNINGS`` (On, **Off**) Enable compiler warnings. -``ENABLE_BOUNDCHECKING`` (On, *Off*) +``ENABLE_BOUNDCHECKING`` (On, **Off**) If building with the Gnu compilers enable STL bound checking by passing -D_GLIBCXX_DEBUG=1 to the compiler. Note, this is a very expensive option at runtime! -``ENABLE_NAN_EXCEPTIONS`` (On, *Off*) +``ENABLE_NAN_EXCEPTIONS`` (On, **Off**) Raise exceptions in the C++ code when floating-point exceptions occur. Gnu compilers only. -``ENABLE_DOCS`` (On, *Off*) +``ENABLE_DOCS`` (On, **Off**) Choose whether or not to build this documentation. ``SPHINX_EXECUTABLE`` Specify where the Sphinx executable is that should be used to build documentation. If not given, assumes the Spheral built Sphinx will be used. -``SPHINX_THEME`` (*sphinx_rtd_theme*) +``SPHINX_THEME`` (**sphinx_rtd_theme**) Give the Sphinx theme to use when generating documentation. Default based on read the docs theme. ``SPHINX_THEME_DIR`` From 77270fc0e5b037bcf84971e9f15b358f7f1c1561 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 5 Sep 2024 11:57:15 -0700 Subject: [PATCH 23/37] Documentation changes --- docs/developer/dev/diagnostic_tools.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 429e8e6a9..921fbee09 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -10,9 +10,9 @@ Spheral uses Caliper to preform diagnostics, such as timing. To enable this func Querying using Caliper ====================== -Caliper is configured and started through the `cali::ConfigManager`. -The `cali::ConfigManager` is wrapped in a `TimerMgr` singleton class, which has a python interface. -`TimerMgr` is initialized and started in the `InitTimers` routine which is called in `commandLine()` in ``src/SimulationControl/SpheralOptionParser.py``. +Caliper is configured and started through the :kbd:`cali::ConfigManager`. +The :kbd:`cali::ConfigManager` is wrapped in a :kbd:`TimerMgr` singleton class, which has a python interface. +:kbd:`TimerMgr` is initialized and started in the :kbd:`InitTimers` routine which is called in :kbd:`commandLine()` in ``src/SimulationControl/SpheralOptionParser.py``. By default, the Caliper configuration is set to ``spot,mem.highwatermark`` and outputs Caliper files (``.cali``). For the default configuration, the Caliper files are named based on what file is being run, for example: :: @@ -36,7 +36,7 @@ Non-default Caliper configurations can be set at the command line using ``--cali python Noh-cylindrical-2d.py --caliperConfig 'runtime-report(output=time.txt),calc.inclusive,region.count' .. note:: - The above configuration produces timing results similar to the previous `Spheral::Timer` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. + The above configuration produces timing results similar to the previous :kbd:`Spheral::Timer` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. Additionally, Caliper timers can be turned off using ``--caliperConfig none``. @@ -58,9 +58,9 @@ So far there are two different types of regions in Spheral, using the following TIME_BEGIN("timer_name") TIME_END("timer_name") -- `TIME_FUNCTION` can be added to the very beginning of a function and creates a region for the entire function using the function's name. `TIME_FUNCTION` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. +- :kbd:`TIME_FUNCTION` can be added to the very beginning of a function and creates a region for the entire function using the function's name. :kbd:`TIME_FUNCTION` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. -- `TIME_BEGIN("timer_name")` and `TIME_END("timer_name")` create a region between the two different calls and use the string (in this case timer_name) as the name. +- :kbd:`TIME_BEGIN("timer_name")` and :kbd:`TIME_END("timer_name")` create a region between the two different calls and use the string (in this case timer_name) as the name. Adding Region Timers in Python From 5c323628014e39cf7353d029b8482e711ec7c23e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 10 Sep 2024 13:59:37 -0700 Subject: [PATCH 24/37] Fixed dev build exporting, improved exporting of find_package TPLs --- cmake/InstallTPLs.cmake | 16 +++++++++++++++ cmake/SetupSpheral.cmake | 4 +--- cmake/spheral/SpheralAddLibs.cmake | 31 ++++++++++++------------------ cmake/spheral_cxx-config.cmake.in | 21 +++++++++++++++----- src/CMakeLists.txt | 9 ++++----- 5 files changed, 49 insertions(+), 32 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index fe12b1fde..a8371a2cd 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -67,9 +67,15 @@ endif() # Find pre-compiled TPLs #----------------------------------------------------------------------------------- +# Any targets that used find package must be added to these lists +set(SPHERAL_FP_TPLS ) +set(SPHERAL_FP_DIRS ) + # Use find_package to get axom (which brings in fmt) and patch fmt find_package(axom REQUIRED NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake) list(APPEND SPHERAL_BLT_DEPENDS axom ) +list(APPEND SPHERAL_FP_TPLS axom) +list(APPEND SPHERAL_FP_DIRS ${axom_DIR}/lib/cmake) # This is a hack to handle transitive issues that come # from using object libraries with newer version of axom @@ -87,6 +93,8 @@ message("----------------------------------------------------------------------- find_package(adiak REQUIRED NO_DEFAULT_PATH PATHS ${adiak_DIR}/lib/cmake/adiak) if(adiak_FOUND) list(APPEND SPHERAL_BLT_DEPENDS adiak::adiak) + list(APPEND SPHERAL_FP_TPLS adiak) + list(APPEND SPHERAL_FP_DIRS ${adiak_DIR}) message("Found Adiak External Package") endif() message("-----------------------------------------------------------------------------") @@ -97,6 +105,8 @@ if (ENABLE_TIMER) find_package(caliper REQUIRED NO_DEFAULT_PATH PATHS ${caliper_DIR}/share/cmake/caliper) if(caliper_FOUND) list(APPEND SPHERAL_BLT_DEPENDS caliper) + list(APPEND SPHERAL_FP_TPLS caliper) + list(APPEND SPHERAL_FP_DIRS ${caliper_DIR}) message("Found Caliper External Package") endif() endif() @@ -118,6 +128,8 @@ if(chai_DIR AND USE_EXTERNAL_CHAI) if (chai_FOUND) message("Found chai External Package.") endif() + list(APPEND SPHERAL_FP_TPLS chai) + list(APPEND SPHERAL_FP_DIRS ${chai_DIR}) else() message("Using chai Submodule.") set(chai_DIR "${SPHERAL_ROOT_DIR}/extern/chai") @@ -126,6 +138,10 @@ else() endif() list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) +list(APPEND SPHERAL_FP_TPLS RAJA umpire) +list(APPEND SPHERAL_FP_DIRS ${raja_DIR} ${umpire_DIR}) +set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) +set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) message("-----------------------------------------------------------------------------") diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index e7c77fa2c..5e6c0d874 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -188,6 +188,4 @@ if (ENABLE_TESTS) DESTINATION ${SPHERAL_TEST_INSTALL_PREFIX}) endif() -if(NOT ENABLE_DEV_BUILD) - include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake) -endif() +include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake) diff --git a/cmake/spheral/SpheralAddLibs.cmake b/cmake/spheral/SpheralAddLibs.cmake index 6e8b9e545..701b1a06c 100644 --- a/cmake/spheral/SpheralAddLibs.cmake +++ b/cmake/spheral/SpheralAddLibs.cmake @@ -46,11 +46,6 @@ function(spheral_add_obj_library package_name obj_list_name) # Install the headers install(FILES ${${package_name}_headers} DESTINATION include/${package_name}) - - if(ENABLE_DEV_BUILD) - install(TARGETS Spheral_${package_name} - DESTINATION lib) - endif() # Append Spheral_${package_name} to the global object list # For example, SPHERAL_OBJ_LIBS or LLNLSPHERAL_OBJ_LIBS set_property(GLOBAL APPEND PROPERTY ${obj_list_name} Spheral_${package_name}) @@ -71,7 +66,7 @@ endfunction() # ---------------------- # INPUT-OUTPUT VARIABLES # ---------------------- -# package_name : REQUIRED : Desired package name +# package_name : REQUIRED : Desired package name (either CXX or LLNLCXX) # _cxx_obj_list : REQUIRED : List of internal targets to include # ----------------------- # OUTPUT VARIABLES TO USE - Made available implicitly after function call @@ -85,14 +80,16 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) get_property(SPHERAL_CXX_DEPENDS GLOBAL PROPERTY SPHERAL_CXX_DEPENDS) # For including files in submodules, currently unused get_property(SPHERAL_SUBMOD_INCLUDES GLOBAL PROPERTY SPHERAL_SUBMOD_INCLUDES) + # Convert package name to lower-case for export target name + string(TOLOWER ${package_name} lower_case_package) + set(export_target_name spheral_${lower_case_package}-targets) - if(ENABLE_SHARED) - # Build shared spheral C++ library - blt_add_library(NAME Spheral_${package_name} - HEADERS ${${package_name}_headers} - SOURCES ${${package_name}_sources} - DEPENDS_ON ${_cxx_obj_list} ${SPHERAL_CXX_DEPENDS} ${SPHERAL_BLT_DEPENDS} - SHARED TRUE) + if(ENABLE_DEV_BUILD) + install(TARGETS ${_cxx_obj_list} + EXPORT ${export_target_name} + DESTINATION lib) + add_library(Spheral_${package_name} INTERFACE) + target_link_libraries(Spheral_${package_name} INTERFACE ${_cxx_obj_list}) else() # Build static spheral C++ library blt_add_library(NAME Spheral_${package_name} @@ -106,7 +103,6 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) set_target_properties(Spheral_${package_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) endif() - ## This cleans up library targets created with object libs. It is turned off as it triggers ## a failure on Werror and pedantic builds. #set(_properties COMPILE_DEFINITIONS LINK_LIBRARIES LINK_OPTIONS INTERFACE_LINK_OPTIONS COMPILE_OPTIONS INTERFACE_COMPILE_OPTIONS) @@ -118,15 +114,12 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) #set_target_properties(Spheral_${package_name} PROPERTIES INTERFACE_LINK_LIBRARIES "") - # Convert package name to lower-case for export target name - string(TOLOWER ${package_name} lower_case_package) - # Install Spheral C++ target and set it as an exportable CMake target install(TARGETS Spheral_${package_name} DESTINATION lib - EXPORT spheral_${lower_case_package}-targets) + EXPORT ${export_target_name}) # Export Spheral target - install(EXPORT spheral_${lower_case_package}-targets DESTINATION lib/cmake) + install(EXPORT ${export_target_name} DESTINATION lib/cmake) # Set the r-path of the C++ lib such that it is independent of the build dir when installed set_target_properties(Spheral_${package_name} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index daf35fdaa..229817de9 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -23,10 +23,12 @@ if(NOT SPHERAL_FOUND) find_package(Python3 COMPONENTS Interpreter Development) set(PYTHON_EXE ${Python3_EXECUTABLE}) endif() - - if(NOT TARGET axom) - find_package(axom REQUIRED QUIET NO_DEFAULT_PATH PATHS ${axom_DIR} ${axom_DIR}/lib ${axom_DIR}/lib/cmake) - endif() + # Loop over TPLs that use find_package + set(SPHERAL_FP_TPLS "@SPHERAL_FP_TPLS@") + set(SPHERAL_FP_DIRS "@SPHERAL_FP_DIRS@") + foreach(tpl dir IN ZIP_LISTS SPHERAL_FP_TPLS SPHERAL_FP_DIRS) + find_package(${tpl} REQUIRED QUIET NO_DEFAULT_PATH PATHS ${dir}) + endforeach() if(NOT TARGET chai) if (@USE_EXTERNAL_CHAI@) set(SPHERAL_CHAI_DIR "@chai_DIR@/lib/cmake/chai") @@ -47,7 +49,16 @@ if(NOT SPHERAL_FOUND) endif() endif() include("${SPHERAL_CXX_INSTALL_PREFIX}/lib/cmake/spheral_cxx-targets.cmake") - + if(@ENABLE_DEV_BUILD) + set(SPHERAL_LIBS @SPHERAL_OBJ_LIBS@) + add_library(Spheral_CXX INTERFACE IMPORTED) + foreach(_lib ${SPHERAL_LIBS}) + set_property(TARGET Spheral_CXX + PROPERTY INTERFACE_LINK_LIBRARIES + ${_lib} + APPEND) + endforeach() + endif() set_property(TARGET Spheral_CXX APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPHERAL_CXX_INCLUDE_DIRS}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 388a9dd44..78588ccf0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,14 +77,13 @@ foreach(_package ${_packages}) add_subdirectory(${_package}) endforeach() +# Retrieve the global list populated in spheral_obj_add_library +get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) if(NOT ENABLE_DEV_BUILD) - # Retrieve the global list populated in spheral_obj_add_library - get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) - set(CXX_sources spheralCXX.cc) - # Must use quotes when passing lists as inputs for functions - spheral_add_cxx_library(CXX "${SPHERAL_OBJ_LIBS}") endif() +# Must use quotes when passing lists as inputs for functions +spheral_add_cxx_library(CXX "${SPHERAL_OBJ_LIBS}") # This calls LLNLSpheralInstallObjs.cmake if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) From 3e45c235c4d711c711e465940f36c88f7840580f Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 10 Sep 2024 16:17:13 -0700 Subject: [PATCH 25/37] Changed axom to be shared by default --- scripts/spack/packages/spheral/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5e143d766..d4b02b08c 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -50,7 +50,9 @@ class Spheral(CachedCMakePackage, CudaPackage): # Zlib fix has been merged into conduit, using develop until next release. depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') - depends_on('axom@0.9.0 ~shared +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom ~shared', when='+cuda') + depends_on('axom +shared', when='~cuda') depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] for ctpl in mpi_tpl_list: From 17d78eedcbd7925f902411abe2b66c14c2ae79df Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 10 Sep 2024 16:32:07 -0700 Subject: [PATCH 26/37] Update Release notes --- RELEASE_NOTES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index df68e6781..bcf4141d2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -15,6 +15,9 @@ Notable changes include: * TPL builds have been split off into a separate Gitlab CI stage to help with timeouts on allocations. * Failed ATS runs are automatically retested once in the Gitlab CI. * Python execute command is centralized in scripts/spheralutils.py now. + * Caliper updated v2.11. + * Adiak added as TPL. + * Created singleton wrapper for cali::ConfigManger and python wrapped Caliper timer and Adiak routines. * Build changes / improvements: * Distributed source directory must always be built now. @@ -23,6 +26,8 @@ Notable changes include: * The FSISPH package is now optional (SPHERAL\_ENABLE\_FSISPH). * The GSPH package is now optional (SPHERAL\_ENABLE\_GSPH). * The SVPH package is now optional (SPHERAL\_ENABLE\_SVPH). + * Cleaner Spheral Spack package. + * ENABLE\_DEV\_BUILD can now export targets properly. * Bug Fixes / improvements: * Wrappers for MPI calls are simplified and improved. From 60f864e8c3b8746c81efe3ba57846bab5378e557 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 11 Sep 2024 09:36:14 -0700 Subject: [PATCH 27/37] Changing axom back to static to see if that fixes the tests --- scripts/spack/packages/spheral/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index d4b02b08c..5e143d766 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -50,9 +50,7 @@ class Spheral(CachedCMakePackage, CudaPackage): # Zlib fix has been merged into conduit, using develop until next release. depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') - depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') - depends_on('axom ~shared', when='+cuda') - depends_on('axom +shared', when='~cuda') + depends_on('axom@0.9.0 ~shared +hdf5 -lua -examples -python -fortran', type='build') depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] for ctpl in mpi_tpl_list: From 29a11f1952801e91af990f3b545a52336635e231 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 11 Sep 2024 13:49:25 -0700 Subject: [PATCH 28/37] Fix bug where ENABLE_SHARED was ignored, added guards for TPL exporting --- cmake/spheral/SpheralAddLibs.cmake | 4 ++-- cmake/spheral_cxx-config.cmake.in | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmake/spheral/SpheralAddLibs.cmake b/cmake/spheral/SpheralAddLibs.cmake index 701b1a06c..2d3fca6d7 100644 --- a/cmake/spheral/SpheralAddLibs.cmake +++ b/cmake/spheral/SpheralAddLibs.cmake @@ -91,12 +91,12 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) add_library(Spheral_${package_name} INTERFACE) target_link_libraries(Spheral_${package_name} INTERFACE ${_cxx_obj_list}) else() - # Build static spheral C++ library + # Build static or shared spheral C++ library blt_add_library(NAME Spheral_${package_name} HEADERS ${${package_name}_headers} SOURCES ${${package_name}_sources} DEPENDS_ON ${_cxx_obj_list} ${SPHERAL_CXX_DEPENDS} ${SPHERAL_BLT_DEPENDS} - SHARED FALSE) + SHARED ${ENABLE_SHARED}) endif() target_include_directories(Spheral_${package_name} SYSTEM PRIVATE ${SPHERAL_SUBMOD_INCLUDES}) if(ENABLE_CUDA) diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 229817de9..138270719 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -27,7 +27,9 @@ if(NOT SPHERAL_FOUND) set(SPHERAL_FP_TPLS "@SPHERAL_FP_TPLS@") set(SPHERAL_FP_DIRS "@SPHERAL_FP_DIRS@") foreach(tpl dir IN ZIP_LISTS SPHERAL_FP_TPLS SPHERAL_FP_DIRS) - find_package(${tpl} REQUIRED QUIET NO_DEFAULT_PATH PATHS ${dir}) + if(NOT TARGET ${tpl}) + find_package(${tpl} REQUIRED QUIET NO_DEFAULT_PATH PATHS ${dir}) + endif() endforeach() if(NOT TARGET chai) if (@USE_EXTERNAL_CHAI@) From a133ba39957f95fdfa87e7a78e8c10dc3a0a5065 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 11 Sep 2024 13:49:59 -0700 Subject: [PATCH 29/37] Making axom always shared --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5e143d766..f03228673 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -50,7 +50,7 @@ class Spheral(CachedCMakePackage, CudaPackage): # Zlib fix has been merged into conduit, using develop until next release. depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') - depends_on('axom@0.9.0 ~shared +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom@0.9.0 +shared +hdf5 -lua -examples -python -fortran', type='build') depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] for ctpl in mpi_tpl_list: From 53e3eb0bb7585e9a3b64baafb63d864511374a4b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 11 Sep 2024 16:53:12 -0700 Subject: [PATCH 30/37] Switched back to using axom~shared+cuda or axom+shared~cuda, added guards to prevent overwriting caliper directory used for exporting --- cmake/InstallTPLs.cmake | 5 ++++- scripts/spack/packages/spheral/package.py | 4 +++- src/SimulationControl/SpheralOptionParser.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index a8371a2cd..9055a0c6f 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -101,7 +101,10 @@ message("----------------------------------------------------------------------- # Use find_package to get caliper if (ENABLE_TIMER) # Save caliper_DIR because it gets overwritten by find_package - set(CONFIG_CALIPER_DIR "${caliper_DIR}" CACHE PATH "Configuration Caliper directory") + if(NOT CONFIG_CALIPER_DIR) + # Only save if it does not exists already + set(CONFIG_CALIPER_DIR "${caliper_DIR}" CACHE PATH "Configuration Caliper directory") + endif() find_package(caliper REQUIRED NO_DEFAULT_PATH PATHS ${caliper_DIR}/share/cmake/caliper) if(caliper_FOUND) list(APPEND SPHERAL_BLT_DEPENDS caliper) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index f03228673..8695500ea 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -50,7 +50,9 @@ class Spheral(CachedCMakePackage, CudaPackage): # Zlib fix has been merged into conduit, using develop until next release. depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') - depends_on('axom@0.9.0 +shared +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom +shared', when='~cuda', type='build') + depends_on('axom ~shared', when='+cuda', type='build') depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] for ctpl in mpi_tpl_list: diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 9955a5299..d846a2cbc 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -68,7 +68,7 @@ def InitTimers(caliper_config, filename): TimerMgr.add(caliper_config) TimerMgr.start() else: - import random, os, sys + import os, sys if (filename): testname = filename else: From 7ba9dbd558e69878094889f2b6eb703f3dc47996 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 16 Sep 2024 11:58:39 -0700 Subject: [PATCH 31/37] Remove unnecessary export config lines, add logic to properly compile and export LLNLSpheral when dev build is turned on --- cmake/spheral_cxx-config.cmake.in | 10 ---------- src/CMakeLists.txt | 20 +++++++++++++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 138270719..2781c4895 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -51,16 +51,6 @@ if(NOT SPHERAL_FOUND) endif() endif() include("${SPHERAL_CXX_INSTALL_PREFIX}/lib/cmake/spheral_cxx-targets.cmake") - if(@ENABLE_DEV_BUILD) - set(SPHERAL_LIBS @SPHERAL_OBJ_LIBS@) - add_library(Spheral_CXX INTERFACE IMPORTED) - foreach(_lib ${SPHERAL_LIBS}) - set_property(TARGET Spheral_CXX - PROPERTY INTERFACE_LINK_LIBRARIES - ${_lib} - APPEND) - endforeach() - endif() set_property(TARGET Spheral_CXX APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPHERAL_CXX_INCLUDE_DIRS}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6afcbc78..2298e9393 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,15 +80,25 @@ foreach(_package ${_packages}) add_subdirectory(${_package}) endforeach() -# Retrieve the global list populated in spheral_obj_add_library -get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) -if(NOT ENABLE_DEV_BUILD) +# For dev builds, we must call LLNLSpheralInstallObjs.cmake +# to ensure LLNLSpheral libraries are added to the Spheral_CXX +# target +if(ENABLE_DEV_BUILD) + # This calls LLNLSpheralInstallObjs.cmake + if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) + include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) + endif() +else() set(CXX_sources spheralCXX.cc) endif() +# Retrieve the global list populated in spheral_obj_add_library +get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) # Must use quotes when passing lists as inputs for functions spheral_add_cxx_library(CXX "${SPHERAL_OBJ_LIBS}") # This calls LLNLSpheralInstallObjs.cmake -if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) - include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) +if(NOT ENABLE_DEV_BUILD) + if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) + include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) + endif() endif() From d1b9f5af0f562600e0c32d9b8f87e444ce5569f0 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 18 Sep 2024 14:11:51 -0700 Subject: [PATCH 32/37] Remove unused cmake variables for timer tests --- tests/unit/CMakeLists.txt | 5 ----- tests/unit/Utilities/testTimers.py.in | 4 ---- 2 files changed, 9 deletions(-) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 85ece892a..9182c51f9 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -1,10 +1,5 @@ add_subdirectory(CXXTests) -if (ENABLE_TIMER) - set(MPI_TIMER_VAR "True") -else() - set(MPI_TIMER_VAR "False") -endif() configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/Utilities/testTimers.py.in" "${SPHERAL_TEST_INSTALL_PREFIX}/tests/unit/Utilities/testTimers.py" diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in index dea75e2f9..53fb21506 100644 --- a/tests/unit/Utilities/testTimers.py.in +++ b/tests/unit/Utilities/testTimers.py.in @@ -14,10 +14,6 @@ import mpi import sys, os, time - -# Set based on ENABLE_TIMERS configure variable -timer_compiler = @MPI_TIMER_VAR@ - commandLine() # Remove cali files from previous test runs From 3279a71a4ba4a4c023104d7cdb6c9c82ff32feae Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 19 Sep 2024 08:25:06 -0700 Subject: [PATCH 33/37] Stop allowing blue os test to fail in CI --- .gitlab/jobs-mpi.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index 2f9ed1556..4e3c2e8b5 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -80,7 +80,6 @@ blueos_cuda_11_gcc_spectrum_build: blueos_cuda_11_gcc_spectrum_test: extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] needs: [blueos_cuda_11_gcc_spectrum_build] - allow_failure: true blueos_cuda_11_gcc_spectrum_cleanup: extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir] From bbc17066146cc832cdfdb6d22adf78c71fc5be8c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 19 Sep 2024 16:29:47 -0700 Subject: [PATCH 34/37] Added ability to specify a caliper configuration json file, remove highwatermark spot configuration as it causes issues with LLNLSpheral tests --- docs/developer/dev/diagnostic_tools.rst | 7 ++++--- src/PYB11/Utilities/TimerMgr.py | 5 +++++ src/SimulationControl/SpheralOptionParser.py | 15 +++++++++++---- src/Utilities/Timer.hh | 8 +++++++- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 921fbee09..d16a2e94c 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -13,7 +13,7 @@ Querying using Caliper Caliper is configured and started through the :kbd:`cali::ConfigManager`. The :kbd:`cali::ConfigManager` is wrapped in a :kbd:`TimerMgr` singleton class, which has a python interface. :kbd:`TimerMgr` is initialized and started in the :kbd:`InitTimers` routine which is called in :kbd:`commandLine()` in ``src/SimulationControl/SpheralOptionParser.py``. -By default, the Caliper configuration is set to ``spot,mem.highwatermark`` and outputs Caliper files (``.cali``). +By default, the Caliper configuration is set to ``spot`` and outputs Caliper files (``.cali``). For the default configuration, the Caliper files are named based on what file is being run, for example: :: @@ -38,9 +38,10 @@ Non-default Caliper configurations can be set at the command line using ``--cali .. note:: The above configuration produces timing results similar to the previous :kbd:`Spheral::Timer` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. -Additionally, Caliper timers can be turned off using ``--caliperConfig none``. +Similarly, a non-default Caliper configuration can be read in from a JSON file using ``--caliperConfigJSON`` and providing the file name. +Lastly, Caliper timers can be turned off using ``--caliperConfig none``. -There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral: +There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorporated into Spheral: * `Configuration basics `_ * `Builtin Configuration `_ diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index 4f61a83aa..6b47214cd 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -34,6 +34,11 @@ def add(self, config_str = "std::string"): "Add a Caliper configuration" return "void" + @PYB11static + def load(self, config_json = "std::string"): + "Load a json file containing Caliper configurations" + return "void" + @PYB11static def default_start(self, testname = "std::string"): "Set the spot Caliper configuration and start the manager" diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index d846a2cbc..7f45e26c0 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -26,12 +26,13 @@ def commandLine(**options): help = "Verbose output -- print all options that were set.") parser.add_argument("--caliperConfig", default="", type=str) parser.add_argument("--caliperFilename", default="", type=str) + parser.add_argument("--caliperConfigJSON", default="", type=str) # Evaluate the command line. args = parser.parse_args() arg_dict = vars(args) if (not TimerMgr.timers_usable()): - if (args.caliperConfig or args.caliperFilename): + if (args.caliperConfig or args.caliperFilename or args.caliperConfigJSON): print("WARNING: Caliper command line inputs provided for "+\ "non-timer install. Reconfigure the install with "+\ "-DENABLE_TIMER=ON to be able to use Caliper timers.") @@ -49,6 +50,8 @@ def commandLine(**options): print(" * caliperConfig = ", args.caliperConfig) if (args.caliperFilename): print(" * caliperFilename = ", args.caliperFilename) + if (args.caliperConfigJSON): + print(" * caliperConfigJSON = ", args.caliperConfigJSON) # Set all the variables. gd = globalFrame().f_globals for key, val in arg_dict.items(): @@ -57,10 +60,14 @@ def commandLine(**options): val = eval(val, gd) gd[key] = val # Initialize timers - InitTimers(args.caliperConfig, args.caliperFilename) + InitTimers(args.caliperConfig, args.caliperFilename, args.caliperConfigJSON) return -def InitTimers(caliper_config, filename): +def InitTimers(caliper_config, filename, caliper_json): + if(caliper_json): + TimerMgr.load(caliper_json) + if(not caliper_config): + raise RuntimeError("SpheralOptionParser: specifying a configuration file without using one of the configurations means no timers are started") off_tests = ["none", "off", "disable", "disabled", "0"] if (caliper_config.lower() in off_tests): return @@ -74,7 +81,7 @@ def InitTimers(caliper_config, filename): else: from datetime import datetime # Append the current day and time to the filename - unique_digits = datetime.now().strftime("_%Y_%m_%d_%H%M%S") + unique_digits = datetime.now().strftime("_%Y_%m_%d_%H%M%S_%f") # Name file based on name of python file being run testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] testname += unique_digits + ".cali" diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 1188ce79b..4ba514b59 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -65,9 +65,13 @@ public: VERIFY2(test, instance().cali_mgr.error_msg()); instance().caliperConfig += config_str; } + static void load(std::string config_json) { + instance().cali_mgr.load(config_json.c_str()); + VERIFY2(!instance().cali_mgr.error(), instance().cali_mgr.error_msg()); + } static void default_start(std::string testname) { if (!testname.empty()) { - std::string default_config = "spot,mem.highwatermark,output=" + testname; + std::string default_config = "spot,output=" + testname; instance().caliperFilename = testname; add(default_config); start(); @@ -91,6 +95,8 @@ public: static bool timers_usable() { return false; } + static void load(std::string) { + } static void default_start(std::string) { } static void add(std::string) { From 6c93ae554db193e5745bb8a44cdbfb1354d68857 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 25 Sep 2024 10:20:40 -0700 Subject: [PATCH 35/37] Updated Caliper diagnostic documentation --- docs/developer/dev/diagnostic_tools.rst | 51 ++++++++++++++++--------- src/PYB11/Utilities/TimerMgr.py | 37 ++++++++---------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index d16a2e94c..95b66b04e 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -1,7 +1,7 @@ -Diagnostics -########### +Code Performance Diagnostics +############################ -Spheral uses Caliper to preform diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ``ENABLE_TIMER=ON``. Otherwise the diagnostic regions are no-ops for improved preformance. +Spheral uses Caliper to preform code diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ``ENABLE_TIMER=ON``. Otherwise, the timing regions are no-ops for improved preformance. :: ./scripts/devtools/host-config-build.py -.cmake -DENABLE_TIMER=ON @@ -10,33 +10,32 @@ Spheral uses Caliper to preform diagnostics, such as timing. To enable this func Querying using Caliper ====================== -Caliper is configured and started through the :kbd:`cali::ConfigManager`. -The :kbd:`cali::ConfigManager` is wrapped in a :kbd:`TimerMgr` singleton class, which has a python interface. -:kbd:`TimerMgr` is initialized and started in the :kbd:`InitTimers` routine which is called in :kbd:`commandLine()` in ``src/SimulationControl/SpheralOptionParser.py``. +Caliper is configured and started through the ``cali::ConfigManager``. +The ``cali::ConfigManager`` is wrapped in a ``TimerMgr`` singleton class, which has a python interface. + +.. note:: + ``TimerMgr`` is initialized and started during ``commandLine()`` in ``src/SimulationControl/SpheralOptionParser.py``. This is because ``commandLine()`` is almost always invoked directly near the start of a problem. However, if ``commandLine()`` is not called, the timers would need to be configured and started directly using the ``TimerMgr`` class. See :ref:`below ` for more details. + By default, the Caliper configuration is set to ``spot`` and outputs Caliper files (``.cali``). For the default configuration, the Caliper files are named based on what file is being run, for example: :: - python Noh-cylindrical-2d.py + python Noh-cylindrical-2d.py -will produce timing files called -:: +will produce a timing file called ``Noh-cylindrical-2d_YEAR_MONTH_DATE_TIME.cali`` where the file name includes the current date and time. - Noh-cylindrical-2d_YEAR_MONTH_DATE_TIME.cali - -where the file name includes the current date and time. The Caliper file name can be specified using the command line :: python Noh-cylindrical-2d.py --caliperFilename 'new_test_name.cali' -Non-default Caliper configurations can be set at the command line using ``--caliperConfig`` like so +Different Caliper configurations can be set at the command line using ``--caliperConfig`` like so :: python Noh-cylindrical-2d.py --caliperConfig 'runtime-report(output=time.txt),calc.inclusive,region.count' .. note:: - The above configuration produces timing results similar to the previous :kbd:`Spheral::Timer` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. + The above configuration produces timing results similar to the previous ``Spheral::Timer`` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. Similarly, a non-default Caliper configuration can be read in from a JSON file using ``--caliperConfigJSON`` and providing the file name. Lastly, Caliper timers can be turned off using ``--caliperConfig none``. @@ -56,12 +55,16 @@ So far there are two different types of regions in Spheral, using the following :: TIME_FUNCTION + +or + +:: TIME_BEGIN("timer_name") TIME_END("timer_name") -- :kbd:`TIME_FUNCTION` can be added to the very beginning of a function and creates a region for the entire function using the function's name. :kbd:`TIME_FUNCTION` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. +- ``TIME_FUNCTION`` can be added to the very beginning of a function and creates a region for the entire function using the function's name. ``TIME_FUNCTION`` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. -- :kbd:`TIME_BEGIN("timer_name")` and :kbd:`TIME_END("timer_name")` create a region between the two different calls and use the string (in this case timer_name) as the name. +- ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case ``timer_name``) as the name. Adding Region Timers in Python @@ -70,9 +73,21 @@ Adding Region Timers in Python Region timers can be added inside the python code using the following function calls: :: - TimerMgr.timer_start("some_function") + TimerMgr.timer_start("timer_name") some_function_call() - TimerMgr.timer_end("some_function") + TimerMgr.timer_end("timer_name") .. note:: IMPORTANT: All timers must have both a start and end call. Otherwise, memory issues will occur. + +.. _manual_caliper: + +Starting Caliper Manually +======================== + +As mentioned above, Caliper (not an individual Caliper timer) is normally configured and started in ``commandLine()`` python routine. However, Caliper can be directly configured and started through the python interface, if desired. This can be done by putting the following into the python file: +:: + + caliper_config = "some_configuration,(output=some_filename.txt)" + TimerMgr.add(caliper_config) + TimerMgr.start() diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py index 6b47214cd..8158110c5 100644 --- a/src/PYB11/Utilities/TimerMgr.py +++ b/src/PYB11/Utilities/TimerMgr.py @@ -6,28 +6,7 @@ @PYB11singleton class TimerMgr: - "Singleton wrapper for CaliperManager. Access through TimerMgr.instance(), ie TimerMgr.instance().start()." - - @PYB11static - @PYB11returnpolicy("reference") - def instance(self): - "Access the singleton instance of the timer manager" - return "TimerMgr&" - - @PYB11static - def timer_start(self, region_name = "std::string"): - "Start custom region Caliper timer, must have corresponding timer_end call" - return "void" - - @PYB11static - def timer_end(self, region_name = "std::string"): - "End custom region Caliper timer" - return "void" - - @PYB11static - def is_started(self): - "Check if ConfigManager has been started" - return "bool" + "Singleton wrapper for CaliperManager. Accesses a C++ singleton object." @PYB11static def add(self, config_str = "std::string"): @@ -73,3 +52,17 @@ def get_filename(self): def timers_usable(self): "Return whether the code has been compiled with timers turned on" return "bool" + @PYB11static + def timer_start(self, region_name = "std::string"): + "Start custom region Caliper timer, must have corresponding timer_end call" + return "void" + + @PYB11static + def timer_end(self, region_name = "std::string"): + "End custom region Caliper timer" + return "void" + + @PYB11static + def is_started(self): + "Check if ConfigManager has been started" + return "bool" From db8dd97085be12e98c1688d0e654882d536fe36c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 25 Sep 2024 10:23:03 -0700 Subject: [PATCH 36/37] Fix documentation bug --- docs/developer/dev/diagnostic_tools.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 95b66b04e..ef28cb38c 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -59,9 +59,11 @@ So far there are two different types of regions in Spheral, using the following or :: + TIME_BEGIN("timer_name") TIME_END("timer_name") + - ``TIME_FUNCTION`` can be added to the very beginning of a function and creates a region for the entire function using the function's name. ``TIME_FUNCTION`` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. - ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case ``timer_name``) as the name. From a249a09efbb426967d25eaef7dd7a4b8545ee18d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 25 Sep 2024 10:26:01 -0700 Subject: [PATCH 37/37] More document fixes --- docs/developer/dev/diagnostic_tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index ef28cb38c..f79cc14b2 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -90,6 +90,6 @@ Starting Caliper Manually As mentioned above, Caliper (not an individual Caliper timer) is normally configured and started in ``commandLine()`` python routine. However, Caliper can be directly configured and started through the python interface, if desired. This can be done by putting the following into the python file: :: - caliper_config = "some_configuration,(output=some_filename.txt)" + caliper_config = "some_configuration(output=some_filename.txt)" TimerMgr.add(caliper_config) TimerMgr.start()