Merge branch 'develop' into experiment/smb

.github/workflows/run.yml

@@ -219,22 +219,23 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
-      - name: Dry run laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
+      - name: Dry run dynamic laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
+          ./bin/benchpark experiment init --dest=laghos-mpi-only laghos
+          ./bin/benchpark setup ./laghos-mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run laghos/mpi-only on LLNL-Ruby-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Ruby-icelake-OmniPath workspace/
+          ./bin/benchpark setup laghos-mpi-only LLNL-Ruby-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

.gitlab/ci/test.yml

@@ -7,13 +7,19 @@
         ARCHCONFIG: LLNL-Dane-DELL-sapphirerapids-OmniPath
 
 test_run:
+  resource_group: $HOST
   stage: test
   tags:
     - $HOST
-    - shell
+    - batch
   <<: *test_clusters
+  variables:
+    SCHEDULER_PARAMETERS: -N 1 -t 1h
   rules:
-    - changes:
+    - if: |
+        $CI_PIPELINE_SOURCE == "merge_request_event" ||
+        $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      changes:
         - .gitlab-ci.yml
         - .gitlab/ci/*
         - experiments/**
@@ -32,4 +38,5 @@ test_run:
     - cd ./workspace/saxpy/openmp/$ARCHCONFIG/workspace/
     - ramble --workspace-dir . --disable-progress-bar --disable-logger workspace setup
     # Run Saxpy Experiments
-    - ramble --workspace-dir . --disable-progress-bar --disable-logger on
+    - ramble --workspace-dir . --disable-progress-bar --disable-logger
+      on --executor '{execute_experiment}' --where '{n_nodes} == 1'

docs/_static/css/custom.css

@@ -89,3 +89,7 @@ a:visited {
 html.writer-html4 .rst-content dl:not(.docutils) > dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt {
     color: #7fa866
 }
+
+.rst-content div[class^="highlight"] pre {
+    max-height: 600px;
+}

docs/benchpark-setup.rst

@@ -11,8 +11,8 @@ To setup an experiment workspace you must first initialize both an experiment an
 Any system or experiment variants are specified at the end of the command as shown below.
 The order of the two init commands does not matter, but they both need to be run before the setup command.::
 
-benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
-benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> <Variant>=<oui/non/value>
+    benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
+    benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> <Variant>=<oui/non/value>
 
 
 Once you have a benchmark experiment to run, along with the programming model to use, and a system to run them on.

docs/conf.py

@@ -7,6 +7,8 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 import subprocess
+import sys
+import os
 
 subprocess.call(
     [
@@ -32,8 +34,12 @@
 extensions = [
     "sphinx_rtd_theme",
     "sphinxcontrib.programoutput",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
 ]
 
+sys.path.insert(0, os.path.abspath("../lib"))
+
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"]
 
 # -- Options for HTML output -------------------------------------------------

docs/index.rst

@@ -48,6 +48,12 @@
    add-an-experiment
    add-a-dryrun
 
+.. toctree::
+   :maxdepth: 2
+   :caption: API Docs
+
+   Benchpark API Docs <source/benchpark>
+
 .. toctree::
    :maxdepth: 1
    :caption: Legacy Workflow

docs/llnl-tutorial.rst

@@ -26,9 +26,9 @@ system specification in Benchpark::
 
     benchpark system init --dest=ruby-system cts cluster=ruby
 
-To run the cuda, strong scaling version of the AMG20223 benchmark, initialize it for experiments::
+To run the openmp, strong scaling version of the AMG20223 benchmark, initialize it for experiments::
 
-    benchpark experiment init --dest=amg2023-benchmark amg2023 cuda=oui strong=oui
+    benchpark experiment init --dest=amg2023-benchmark amg2023 openmp=oui
 
 Then setup the workspace directory for the system and experiment together::
 
@@ -44,22 +44,22 @@ Then setup the Ramble experiment workspace, this builds all software and may tak
     cd ./workspace/amg2023-benchmark/Cts-6d48f81/workspace/
     ramble --workspace-dir . --disable-progress-bar workspace setup
 
-Next, we run the Saxpy experiments, which will launch jobs through the
-scheduler on Tioga::
+Next, we run the AMG2023 experiments, which will launch jobs through the
+scheduler on the CTS system::
 
     ramble --workspace-dir . --disable-progress-bar on
 
 ------
 Tioga
 ------
 
-This second tutorial will guide you through the process of using the cuda 
+This second tutorial will guide you through the process of using the ROCm
 version of the Saxpy benchmark on Tioga. 
 The parameters for initializing the system are slightly different due to the 
 different variants defined for the system. For example, the variant ``~gtl`` turns off gtl-enabled MPI, ``+gtl`` turns it on::
 
     benchpark system init --dest=tioga-system tioga ~gtl
-    benchpark experiment init --dest=saxpy-benchmark saxpy cuda=oui
+    benchpark experiment init --dest=saxpy-benchmark saxpy rocm=oui
     benchpark setup ./saxpy-benchmark ./tioga-system workspace/
     . workspace/setup.sh
     cd ./workspace/saxpy-benchmark/Tioga-975af3c/workspace/

docs/modifiers.rst

@@ -99,7 +99,7 @@ setup step::
 
     benchpark experiment init --dest=</path/to/experiment_root> caliper=<caliper_modifier> </output/path/to/workspace> 
 
-Valid values for ``<caliper>`` are found in the **Caliper Variant**
+Valid values for ``<caliper_variant>`` are found in the **Caliper Variant**
 column of the table below.  Benchpark will link the experiment to Caliper,
 and inject appropriate Caliper configuration at runtime.  After the experiments
 in the workspace have completed running, a ``.cali`` file

experiments/laghos/experiment.py

@@ -0,0 +1,80 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.scaling import StrongScaling
+from benchpark.expr.builtin.caliper import Caliper
+
+
+class Laghos(
+    Experiment,
+    StrongScaling,
+    Caliper,
+):
+
+    variant(
+        "workload",
+        default="triplept",
+        description="triplept or other problem",
+    )
+
+    variant(
+        "version",
+        default="develop",
+        description="app version",
+    )
+
+    def compute_applications_section(self):
+        # TODO: Replace with conflicts clause
+        scaling_modes = {
+            "strong": self.spec.satisfies("strong=oui"),
+            "single_node": self.spec.satisfies("single_node=oui"),
+        }
+
+        scaling_mode_enabled = [key for key, value in scaling_modes.items() if value]
+        if len(scaling_mode_enabled) != 1:
+            raise BenchparkError(
+                f"Only one type of scaling per experiment is allowed for application package {self.name}"
+            )
+
+        # Number of initial nodes
+        num_nodes = {"n_nodes": 1}
+
+        if self.spec.satisfies("single_node=oui"):
+            for pk, pv in num_nodes.items():
+                self.add_experiment_variable(pk, pv, True)
+        elif self.spec.satisfies("strong=oui"):
+            scaled_variables = self.generate_strong_scaling_params(
+                {tuple(num_nodes.keys()): list(num_nodes.values())},
+                int(self.spec.variants["scaling-factor"][0]),
+                int(self.spec.variants["scaling-iterations"][0]),
+            )
+            for pk, pv in scaled_variables.items():
+                self.add_experiment_variable(pk, pv, True)
+
+        self.add_experiment_variable(
+            "n_ranks", "{sys_cores_per_node} * {n_nodes}", True
+        )
+
+    def compute_spack_section(self):
+        # get package version
+        app_version = self.spec.variants["version"][0]
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+
+        # set package spack specs
+        # empty package_specs value implies external package
+        self.add_spack_spec(system_specs["mpi"])
+        # self.add_spack_spec(system_specs["blas"])
+
+        self.add_spack_spec(
+            self.name, [f"laghos@{app_version} +metis", system_specs["compiler"]]
+        )

lib/benchpark/directives.py

@@ -116,8 +116,10 @@ def variant(
     sticky: bool = False,
 ):
     """Define a variant.
+
     Can specify a default value as well as a text description.
-    Args:
+
+    Arguments:
         name: Name of the variant
         default: Default value for the variant, if not specified otherwise the default will be
             False for a boolean variant and 'nothing' for a multi-valued variant
@@ -130,6 +132,7 @@ def variant(
             if the group doesn't meet the additional constraints
         when: Optional condition on which the variant applies
         sticky: The variant should not be changed by the concretizer to find a valid concrete spec
+
     Raises:
         DirectiveError: If arguments passed to the directive are invalid
     """

lib/benchpark/repo.py

@@ -151,7 +151,7 @@ def set_path(repo, object_type=default_type):
 def additional_repository(repository, object_type=default_type):
     """Adds temporarily a repository to the default one.
 
-    Args:
+    Parameters:
         repository: repository to be added
     """
     paths[object_type].put_first(repository)
@@ -163,8 +163,8 @@ def additional_repository(repository, object_type=default_type):
 def use_repositories(*paths_and_repos, object_type=default_type):
     """Use the repositories passed as arguments within the context manager.
 
-    Args:
-        *paths_and_repos: paths to the repositories to be used, or
+    Parameters:
+        `*paths_and_repos`: paths to the repositories to be used, or
             already constructed Repo objects
 
     Returns:

lib/benchpark/spec.py

@@ -623,11 +623,11 @@ def tokens(self) -> List[Token]:
     def next_spec(self) -> Optional[Spec]:
         """Return the next spec parsed from text.
 
-        Args:
+        Parameters:
             initial_spec: object where to parse the spec. If None a new one
                 will be created.
 
-        Return
+        Returns:
             The spec that was parsed
         """
         if not self.ctx.next_token:

lib/benchpark/variant.py

@@ -24,7 +24,7 @@ def __init__(
     ):
         """Initialize a package variant.
 
-        Args:
+        Parameters:
             name (str): name of the variant
             default (str): default value for the variant in case
                 nothing has been specified
@@ -74,12 +74,13 @@ def validate_values(self, variant_values, pkg_cls=None):
         """Validate a variant spec against this package variant. Raises an
         exception if any error is found.
 
-        Args:
+        Parameters:
             vspec_values (tuple): values to be validated
             pkg_cls (spack.package_base.PackageBase): the package class
                 that required the validation, if available
 
-        Raises: Exception
+        Raises:
+            Exception
         """
         # If the value is exclusive there must be at most one
         if not self.multi and len(variant_values) != 1:

repo/laghos/application.py

@@ -19,9 +19,29 @@ class Laghos(ExecutableApplication):
             'lagrangian','spatial-discretization','unstructured-grid',
             'network-latency-bound','network-collectives','unstructured-grid']
 
-    executable('p', 'laghos -p 3 -m {laghos}/data/box01_hex.mesh -rs 5 -ms 500', use_mpi=True)
-
-    workload('problem', executables=['p'])
+    executable('prob', 'laghos -p {problem} -m {mesh} -rs {rs} -rp {rp} -ms {ms}', use_mpi=True)
+
+    workload('triplept', executables=['prob'])
+
+    workload_variable('mesh', default='{laghos}/data/box01_hex.mesh',
+            description='mesh file',
+            workloads=['triplept'])
+
+    workload_variable('problem', default='3',
+            description='problem number',
+            workloads=['triplept'])
+
+    workload_variable('rs', default='5',
+            description='number of serial refinements',
+            workloads=['triplept'])
+
+    workload_variable('rp', default='0',
+            description='number of parallel refinements',
+            workloads=['triplept'])
+
+    workload_variable('ms', default='500',
+            description='max number of steps',
+            workloads=['triplept'])
 
     figure_of_merit('Major kernels total time',
                     log_file='{experiment_run_dir}/{experiment_name}.out',

repo/laghos/package.py

@@ -33,7 +33,8 @@ class Laghos(MakefilePackage):
     depends_on("caliper", when="+caliper")
     depends_on("adiak", when="+caliper")
 
-    depends_on("mfem@develop", when="@develop")
+    depends_on("[email protected]+optimize+pic+shared", when="@develop")
+    depends_on("mfem@develop^[email protected]+optimize+pic+shared", when="@develop")
     depends_on("[email protected]:", when="@3.1")
     depends_on("[email protected]:4.1", when="@3.0")
     # Recommended mfem version for laghos v2.0 is: ^[email protected]

systems/cts/externals/base/00-packages.yaml

@@ -47,4 +47,6 @@ packages:
     buildable: false
     externals:
     - spec: [email protected]
-      prefix: /usr/tce/packages/fftw/fftw-3.3.10
+      prefix: /usr/tce/packages/fftw/fftw-3.3.10
+  zlib-api:                                                                     
+    require: zlib

systems/genericx86/externals/base/00-packages.yaml

@@ -0,0 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  zlib-api:                                                                     
+    require: zlib

systems/sierra/externals/base/00-packages.yaml

@@ -49,3 +49,5 @@ packages:
       buildable: false
   mpi:
     buildable: false
+  zlib-api:                                                                     
+      require: zlib

systems/tioga/externals/base/00-packages.yaml

@@ -184,3 +184,5 @@ packages:
     buildable: false
   rocsolver:
     buildable: false
+  zlib-api:                                                                     
+    require: zlib