benchpark setup: easier referencing of dynamically generated systems (

#450)

* command for generating system directory name used by benchpark setup

* update all dry-runs to use 'benchpark system id'

* add simlink that matches the name of the system config dir used in benchpark setup

* demo use of symlinks for systems (which works like experiments)

* style fix

* remove unused import

.github/workflows/run.yml

@@ -177,9 +177,10 @@ jobs:
         run: |
           ./bin/benchpark system init --dest=tioga-system tioga rocm=551 compiler=cce ~gtl
           ./bin/benchpark setup kripke/rocm ./tioga-system workspace/
+          system_id=$(./bin/benchpark system id ./tioga-system)
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/kripke/rocm/Tioga-975af3c/workspace \
+            --workspace-dir "workspace/kripke/rocm/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
@@ -198,23 +199,25 @@ jobs:
       - name: Dry run dynamic saxpy/rocm with dynamic Tioga
         run: |
           ./bin/benchpark system init --dest=tioga-system2 tioga rocm=551 compiler=cce ~gtl
+          system_id=$(./bin/benchpark system id ./tioga-system2)
           ./bin/benchpark experiment init --dest=saxpy-rocm2 saxpy+rocm
           ./bin/benchpark setup ./saxpy-rocm2 ./tioga-system2 workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-rocm2/Tioga-975af3c/workspace \
+            --workspace-dir "workspace/saxpy-rocm2/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run dynamic saxpy/cuda with dynamic Sierra
         run: |
           ./bin/benchpark system init --dest=sierra-system sierra cuda=10-1-243 compiler=xl
+          system_id=$(./bin/benchpark system id ./sierra-system)
           ./bin/benchpark experiment init --dest=saxpy-cuda saxpy+cuda
           ./bin/benchpark setup ./saxpy-cuda ./sierra-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-cuda/Sierra-bdc4915/workspace \
+            --workspace-dir "workspace/saxpy-cuda/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
@@ -334,23 +337,25 @@ jobs:
       - name: Dry run dynamic saxpy/openmp with dynamic CTS ruby
         run: |
           ./bin/benchpark system init --dest=ruby-system cts cluster=ruby
+          system_id=$(./bin/benchpark system id ./ruby-system)
           ./bin/benchpark experiment init --dest=saxpy-openmp saxpy+openmp
           ./bin/benchpark setup ./saxpy-openmp ./ruby-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-openmp/Cts-6d48f81/workspace \
+            --workspace-dir "workspace/saxpy-openmp/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
             
       - name: Dry run dynamic saxpy/openmp with dynamic CTS dane
         run: |
           ./bin/benchpark system init --dest=dane-system cts cluster=dane
+          system_id=$(./bin/benchpark system id ./dane-system)
           ./bin/benchpark experiment init --dest=saxpy-openmp2 saxpy+openmp
           ./bin/benchpark setup ./saxpy-openmp2 ./dane-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-openmp2/Cts-2c51a80/workspace \
+            --workspace-dir "workspace/saxpy-openmp2/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
@@ -362,7 +367,7 @@ jobs:
           ./bin/benchpark setup ./saxpy-openmp3 ./magma-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-openmp3/Cts-54a5761/workspace \
+            --workspace-dir "workspace/saxpy-openmp3/magma-system/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
@@ -374,18 +379,19 @@ jobs:
           ./bin/benchpark setup ./saxpy-omp-generic ./x86-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-omp-generic/Genericx86-040898b/workspace \
+            --workspace-dir "workspace/saxpy-omp-generic/x86-system/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run dynamic ior/mpi with dynamic CTS ruby
         run: |
+          system_id=$(./bin/benchpark system id ./ruby-system)
           ./bin/benchpark experiment init --dest=ior-mpi ior
           ./bin/benchpark setup ./ior-mpi ./ruby-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/ior-mpi/Cts-6d48f81/workspace \
+            --workspace-dir "workspace/ior-mpi/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

lib/benchpark/cmd/setup.py

@@ -10,8 +10,6 @@
 import shutil
 import sys
 
-import yaml
-
 import benchpark.paths
 from benchpark.accounting import (
     benchpark_experiments,
@@ -20,6 +18,7 @@
 )
 from benchpark.debug import debug_print
 from benchpark.runtime import RuntimeResources
+import benchpark.system
 
 
 # Note: it would be nice to vendor spack.llnl.util.link_tree, but that
@@ -100,12 +99,8 @@ def benchpark_check_system(arg_str):
     if cfg_path.is_dir():
         system_id_path = cfg_path / "system_id.yaml"
         if system_id_path.exists():
-            with open(system_id_path, "r") as f:
-                data = yaml.safe_load(f)
-            name = data["system"]["name"]
-            spec_hash = data["system"]["config-hash"]
-            system_id = f"{name}-{spec_hash[:7]}"
-            return system_id, cfg_path
+            system_id = benchpark.system.unique_dir_for_description(cfg_path)
+            return system_id, cfg_path.name, cfg_path
 
     # If it's not a directory, it might be a shorthand that refers
     # to a pre-constructed config
@@ -121,7 +116,7 @@ def benchpark_check_system(arg_str):
     configs_src_dir = (
         benchpark.paths.benchpark_root / "legacy" / "systems" / str(arg_str)
     )
-    return arg_str, configs_src_dir
+    return arg_str, None, configs_src_dir
 
 
 def benchpark_check_modifier(arg_str):
@@ -154,7 +149,7 @@ def command(args):
     debug_print(f"source_dir = {source_dir}")
     experiment_id, experiment_src_dir = benchpark_check_experiment(args.experiment)
     debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment_id}")
-    system_id, configs_src_dir = benchpark_check_system(args.system)
+    system_id, simple_system_name, configs_src_dir = benchpark_check_system(args.system)
     debug_print(f"specified system = {system_id}")
     debug_print(f"specified modifier = {modifier}")
     benchpark_check_modifier(modifier)
@@ -172,6 +167,10 @@ def command(args):
             sys.exit(1)
 
     workspace_dir.mkdir(parents=True)
+    if simple_system_name:
+        os.symlink(
+            workspace_dir, experiments_root / str(experiment_id) / simple_system_name
+        )
 
     ramble_workspace_dir = workspace_dir / "workspace"
     ramble_configs_dir = ramble_workspace_dir / "configs"

lib/benchpark/cmd/system.py

@@ -45,6 +45,10 @@ def system_list(args):
     raise NotImplementedError("'benchpark system list' is not available")
 
 
+def system_id(args):
+    print(benchpark.system.unique_dir_for_description(args.system_dir))
+
+
 def setup_parser(root_parser):
     system_subparser = root_parser.add_subparsers(dest="system_subcommand")
 
@@ -58,11 +62,15 @@ def setup_parser(root_parser):
 
     system_subparser.add_parser("list")
 
+    id_parser = system_subparser.add_parser("id")
+    id_parser.add_argument("system_dir")
+
 
 def command(args):
     actions = {
         "init": system_init,
         "list": system_list,
+        "id": system_id,
     }
     if args.system_subcommand in actions:
         actions[args.system_subcommand](args)

lib/benchpark/system.py

@@ -8,6 +8,7 @@
 import os
 import pathlib
 import sys
+import yaml
 
 import benchpark.paths
 from benchpark.directives import ExperimentSystemBase
@@ -188,3 +189,12 @@ def variables_yaml(self):
   batch_submit: "placeholder"
   mpi_command: "placeholder"
 """
+
+
+def unique_dir_for_description(system_dir):
+    system_id_path = os.path.join(system_dir, "system_id.yaml")
+    with open(system_id_path, "r") as f:
+        data = yaml.safe_load(f)
+    name = data["system"]["name"]
+    spec_hash = data["system"]["config-hash"]
+    return f"{name}-{spec_hash[:7]}"