Adding Sandia microbenchmarks experiment class (#431)

* initial commit for progress on sandia microbenchmarks

* package changes

* experiment class and package fixes

* new experiment class, rma version

* lint

* lint

* more lint

* Delete experiments/smb/mpi directory

remove old format

* lint

* removing hidden tab

* lint

* adding space between lines

* adding dryruns

* Indentation

* removing bad indent in dryrun

---------

Co-authored-by: august-knox <[email protected]>
Co-authored-by: pearce8 <[email protected]>

.github/workflows/run.yml

@@ -539,6 +539,30 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
+      - name: Dry run dynamic smb-mpi_overhead with dynamic CTS ruby
+        run: |
+          ./bin/benchpark experiment init --dest=smb-mpi_overhead smb
+          ./bin/benchpark setup ./smb-mpi_overhead ./ruby-system workspace/
+          system_id=$(./bin/benchpark system id ./ruby-system)
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/smb-mpi_overhead/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic smb-msgrate with dynamic CTS ruby
+        run: |
+          ./bin/benchpark experiment init --dest=smb-msgrate smb workload=msgrate
+          ./bin/benchpark setup ./smb-msgrate ./ruby-system workspace/
+          system_id=$(./bin/benchpark system id ./ruby-system)
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/smb-msgrate/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
       - name: Dry run dynamic phloem/mpi with dynamic CTS ruby
         run: |
           ./bin/benchpark experiment init --dest=phloem-mpi phloem
@@ -561,4 +585,4 @@ jobs:
             --workspace-dir "workspace/osu-micro-benchmarks-mpi-only/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
-            workspace setup --dry-run
+            workspace setup --dry-run

experiments/smb/experiment.py

@@ -0,0 +1,45 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.scaling import StrongScaling
+
+
+class Smb(Experiment, StrongScaling):
+    variant(
+        "workload",
+        default="mpi_overhead",
+        values=("mpi_overhead", "msgrate", "rma_mt"),
+        description="workload",
+    )
+
+    variant(
+        "version",
+        default="master",
+        description="app version",
+    )
+
+    def compute_applications_section(self):
+        if self.spec.satisfies("workload=mpi_overhead"):
+            self.add_experiment_variable("n_ranks", "2")
+        elif self.spec.satisfies("workload=msgrate") or self.spec.satisfies(
+            "workload=rma_mt"
+        ):
+            self.add_experiment_variable("n_nodes", "1")
+            self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}")
+
+    def compute_spack_section(self):
+        # get package version
+        app_version = self.spec.variants["version"][0]
+
+        spec_string = f"smb@{app_version} +mpi"
+        if self.spec.satisfies("workload=rma_mt"):
+            spec_string += "+rma"
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+        self.add_spack_spec(system_specs["mpi"])
+        self.add_spack_spec(self.name, [spec_string, system_specs["compiler"]])

repo/smb/application.py

@@ -0,0 +1,51 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+from ramble.appkit import *
+
+
+class Smb(ExecutableApplication):
+    """Sandia microbenchmarks"""
+    name = "Sandia microbenchmarks"
+
+    executable('p1', 'mpi_overhead -v', use_mpi=True)
+    executable('p2', 'msgrate -n {ppn}', use_mpi=True)
+
+    workload('mpi_overhead', executables=['p1'])
+    workload('msgrate', executables=['p2'])
+    workload('rma_mt', executables=['p2'])
+
+    workload_variable('ppn', default='1',
+                   description='Number of procs per node',
+                   workloads=['msgrate', 'rma_mt'])
+
+    figure_of_merit('single direction',
+                   log_file='{experiment_run_dir}/{experiment_name}.out',
+                   fom_regex=r'single direction:\s+(?P<fom>[0-9]+\.[0-9]*)',
+                  group_name='fom', units='')
+    #TODO:fix this FOM. Not sure what's causing it to not detect
+    figure_of_merit('overhead',
+                    log_file='{experiment_run_dir}/{experiment_name}.out',
+                    fom_regex=r'(?:[0-9]+\.?[0-9]* +){4}(?P<fom>[0-9]+\.[0-9]*)',
+                    #fom_regex=r'avail\(%\)(?:\s|\t)*\n\s*(?:[0-9]+\.*[0-9]*\s*){4}(?P<fom>[0-9]+\.*[0-9]*)',
+                   group_name='fom', units='')
+
+    figure_of_merit('pair based',
+                    log_file='{experiment_run_dir}/{experiment_name}.out',
+                    fom_regex=r'pair-based:\s+(?P<fom>[0-9]+\.[0-9]*)',
+                   group_name='fom', units='')
+
+    figure_of_merit('pre-post',
+                    log_file='{experiment_run_dir}/{experiment_name}.out',
+                    fom_regex=r'\s*pre-post:\s+(?P<fom>[0-9]+\.[0-9]*)',
+                   group_name='fom', units='')
+
+    figure_of_merit('all-start',
+                    log_file='{experiment_run_dir}/{experiment_name}.out',
+                    fom_regex=r'\s*all-start:\s+(?P<fom>[0-9]+\.[0-9]*)',
+                   group_name='fom', units='')
+    #success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out')

repo/smb/package.py

@@ -0,0 +1,46 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+from spack.package import *
+import llnl.util.filesystem as fs
+import inspect
+class Smb(MakefilePackage):
+    tags = []
+
+    url = "https://github.com/sandialabs/SMB/archive/refs/tags/1.1.tar.gz"
+    git = "https://github.com/sandialabs/SMB"
+
+    maintainers("knox10")
+
+    version("master", branch="master")
+
+    variant("mpi", default=False, description="Build with MPI support")
+    variant("rma", default=False, description="Build RMA-MT variant")    
+    depends_on("mpi", when="+mpi")
+    build_directory = ["src/mpi_overhead"]
+
+    def edit(self, spec, prefix):
+        if "+rma" in spec:
+            makefile = FileFilter("src/rma_mt_mpi/Makefile")
+            makefile.filter('CC=cc', "CC = {0}".format(spec["mpi"].mpicc))
+    #TODO: add shm variant
+    def build(self, spec, prefix):
+        if "+rma" in spec: 
+            self.build_directory.append("src/rma_mt_mpi")
+        else:
+            self.build_directory.append("src/msgrate")
+
+        for path in self.build_directory:
+            with fs.working_dir(path):
+                make()
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        mkdir(prefix.doc)
+        install("src/mpi_overhead/mpi_overhead", prefix.bin)
+        install("src/mpi_overhead/README", prefix.doc)
+        if "+rma" in spec:
+            install("src/rma_mt_mpi/msgrate", prefix.bin)
+        else:
+            install("src/msgrate/msgrate", prefix.bin)
+            install("src/msgrate/README", prefix.doc)