[develop] Port SRW to Gaea C6 (#1163)

Enable SRW for new machine Gaea-C6.
UFS_SRW_data copied to /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data location.

---------

Co-authored-by: michael.lueken <[email protected]>

.cicd/Jenkinsfile

@@ -12,10 +12,10 @@ pipeline {
     parameters {
         // Allow job runner to filter based on platform
         // Use the line below to enable all PW clusters
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6','hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable the PW AWS cluster
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Workflow Wrapper test depth {0..9}, 0=none, 1=simple, 9=all [default]
@@ -103,7 +103,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'derecho', 'gaea', 'gaea-c6', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -117,7 +117,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'derecho', 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'derecho', 'gaea', 'gaea-c6', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {

.cicd/scripts/wrapper_srw_ftest.sh

@@ -28,6 +28,13 @@ if [[ "${SRW_PLATFORM}" == gaea ]]; then
     sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "gaea" "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh
 fi
 
+if [[ "${SRW_PLATFORM}" == gaea-c6 ]]; then
+    sed -i '15i #SBATCH --clusters=c6' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+    sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+    sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+    sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "gaea-c6" "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh
+fi
+
 if [[ "${SRW_PLATFORM}" == hera ]]; then
     if [[ "${SRW_COMPILER}" == gnu ]]; then
         sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh

.github/PULL_REQUEST_TEMPLATE

@@ -32,6 +32,7 @@
 
 - [ ] derecho.intel
 - [ ] gaea.intel
+- [ ] gaea-c6.intel
 - [ ] hera.gnu
 - [ ] hera.intel
 - [ ] hercules.intel

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 33b3c18
+hash = e119370
 local_path = sorc/ufs-weather-model
 required = True
 

devbuild.sh

@@ -285,7 +285,7 @@ set -eu
 # automatically determine compiler
 if [ -z "${COMPILER}" ] ; then
   case ${PLATFORM} in
-    jet|hera|gaea) COMPILER=intel ;;
+    jet|hera|gaea|gaea-c6) COMPILER=intel ;;
     orion) COMPILER=intel ;;
     wcoss2) COMPILER=intel ;;
     cheyenne) COMPILER=intel ;;

doc/ContribGuide/contributing.rst

@@ -229,6 +229,7 @@ Here is the template that is provided when developers click "Create pull request
 
    - [ ] derecho.intel
    - [ ] gaea.intel
+   - [ ] gaea-c6.intel
    - [ ] hera.gnu
    - [ ] hera.intel
    - [ ] hercules.intel

doc/ContribGuide/testing.rst

@@ -49,9 +49,7 @@ Additionally, users can potentially access the directories where the Jenkins tes
    * - Derecho
      - /glade/derecho/scratch/epicufsrt/jenkins/workspace
    * - Gaea
-     - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea
-   * - Gaea C5
-     - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea-c5
+     - /gpfs/f5/epic/scratch/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea
    * - Hera (Intel)
      - /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#__2/hera
    * - Hera (GNU)

doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst

@@ -76,8 +76,8 @@ On most Level 1 systems, a container named ``ubuntu20.04-intel-ue-1.4.1-srw-dev.
      - File Location
    * - Derecho [#fn]_
      - /glade/work/epicufsrt/contrib/containers
-   * - Gaea [#fn]_
-     - /lustre/f2/dev/role.epic/containers
+   * - Gaea-C5 [#fn]_
+     - /gpfs/f5/epic/world-shared/containers
    * - Hera
      - /scratch1/NCEPDEV/nems/role.epic/containers
    * - Jet

doc/UsersGuide/BuildingRunningTesting/Quickstart.rst

@@ -42,7 +42,7 @@ For a detailed explanation of how to build and run the SRW App on any supported
             
          ./devbuild.sh --platform=<machine_name>
 
-      where ``<machine_name>`` is replaced with the name of the user's platform/system. Valid values include: ``derecho`` | ``gaea`` | ``hera`` | ``hercules`` | ``jet`` | ``linux`` | ``macos`` | ``noaacloud`` | ``orion`` | ``wcoss2``
+      where ``<machine_name>`` is replaced with the name of the user's platform/system. Valid values include: ``derecho`` | ``gaea`` | ``gaea-c6`` | ``hera`` | ``hercules`` | ``jet`` | ``linux`` | ``macos`` | ``noaacloud`` | ``orion`` | ``wcoss2``
 
       For additional details, see :numref:`Section %s <DevBuild>`, or view :numref:`Section %s <CMakeApproach>` to try the CMake build approach instead. 
 
@@ -72,7 +72,7 @@ For a detailed explanation of how to build and run the SRW App on any supported
       
       Users will need to open the ``config.yaml`` file and adjust the experiment parameters in it to suit the needs of their experiment (e.g., date, grid, physics suite). At a minimum, users need to modify the ``MACHINE`` parameter. In most cases, users will need to specify the ``ACCOUNT`` parameter and the location of the experiment data (see :numref:`Section %s <Data>` for Level 1 system default locations). 
 
-      For example, a user on Gaea might adjust or add the following fields to run the 12-hr "out-of-the-box" case on Gaea using prestaged system data and :term:`cron` to automate the workflow: 
+      For example, a user on Gaea-C5 might adjust or add the following fields to run the 12-hr "out-of-the-box" case on Gaea-C5 using prestaged system data and :term:`cron` to automate the workflow: 
 
       .. code-block:: console
          

doc/UsersGuide/BuildingRunningTesting/RunSRW.rst

@@ -55,8 +55,10 @@ The SRW App requires input files to run. These include static datasets, initial
      - File location
    * - Derecho
      - /glade/work/epicufsrt/contrib/UFS_SRW_data/|data|/input_model_data
-   * - Gaea
+   * - Gaea-C5
      - /gpfs/f5/epic/world-shared/UFS_SRW_data/|data|/input_model_data/
+   * - Gaea-C6
+     - /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/|data|/input_model_data/
    * - Hera
      - /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/|data|/input_model_data/
    * - Hercules

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -30,7 +30,7 @@ If non-default parameters are selected for the variables in this section, they s
    Setting ``RUN_ENVIR`` to "community" is recommended in most cases for users who are not running in NCO's production environment. Valid values: ``"nco"`` | ``"community"``
 
 ``MACHINE``: (Default: "BIG_COMPUTER")
-   The machine (a.k.a. platform or system) on which the workflow will run. Currently supported platforms are listed on the :srw-wiki:`SRW App Wiki page <Supported-Platforms-and-Compilers>`. When running the SRW App on any ParallelWorks/NOAA Cloud system, use "NOAACLOUD" regardless of the underlying system (AWS, GCP, or Azure). Valid values: ``"HERA"`` | ``"ORION"`` | ``"HERCULES"`` | ``"JET"`` | ``"CHEYENNE"`` | ``"DERECHO"`` | ``"GAEA"`` |  ``"NOAACLOUD"`` | ``"STAMPEDE"`` | ``"ODIN"`` | ``"MACOS"`` | ``"LINUX"`` | ``"SINGULARITY"`` | ``"WCOSS2"`` (Check ``ufs-srweather-app/ush/valid_param_vals.yaml`` for the most up-to-date list of supported platforms.)
+   The machine (a.k.a. platform or system) on which the workflow will run. Currently supported platforms are listed on the :srw-wiki:`SRW App Wiki page <Supported-Platforms-and-Compilers>`. When running the SRW App on any ParallelWorks/NOAA Cloud system, use "NOAACLOUD" regardless of the underlying system (AWS, GCP, or Azure). Valid values: ``"HERA"`` | ``"ORION"`` | ``"HERCULES"`` | ``"JET"`` | ``"CHEYENNE"`` | ``"DERECHO"`` | ``"GAEA"`` | ``"GAEA-C6"`` |  ``"NOAACLOUD"`` | ``"STAMPEDE"`` | ``"ODIN"`` | ``"MACOS"`` | ``"LINUX"`` | ``"SINGULARITY"`` | ``"WCOSS2"`` (Check ``ufs-srweather-app/ush/valid_param_vals.yaml`` for the most up-to-date list of supported platforms.)
 
    .. hint::
       Users who are NOT on a named, supported Level 1 or 2 platform will need to set the ``MACHINE`` variable to ``LINUX`` or ``MACOS``. To combine use of a Linux or MacOS platform with the Rocoto workflow manager, users will also need to set ``WORKFLOW_MANAGER: "rocoto"`` in the ``platform:`` section of ``config.yaml``. This combination will assume a Slurm batch manager when generating the XML. 

doc/tables/Tests.csv

@@ -52,8 +52,8 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_
 ,yes,pregen_grid_orog_sfc_climo,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,17,
 ,yes,specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2021061500,6,6,19,
 ,yes,specify_template_filenames,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,28,
-,hera/jet/orion/gaea,get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems,RRFS_CONUS_3km,FV3_HRRR,GEFS,GEFS,2022040400,6,805,45,
-,hera/jet/orion/gaea,get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,*2 days ago*,6,12,40,Retrieves online data and runs a forecast valid 00z 2 days previous to the test date
+,hera/jet/orion/gaea/gaea-c6,get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems,RRFS_CONUS_3km,FV3_HRRR,GEFS,GEFS,2022040400,6,805,45,
+,hera/jet/orion/gaea/gaea-c6,get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,*2 days ago*,6,12,40,Retrieves online data and runs a forecast valid 00z 2 days previous to the test date
 ,hera/jet,custom_ESGgrid_Great_Lakes_snow_8km,*custom*,FV3_RAP,RAP,RAP,2023021700,6,,,
 ,hera/jet,get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019061200,6,6,19,
 ,hera/jet,get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019061200,6,10,23,

etc/lmod-setup.csh

@@ -40,6 +40,9 @@ else if ( "$L_MACHINE" == singularity ) then
 else if ( "$L_MACHINE" == gaea ) then
    module reset
 
+else if ( "$L_MACHINE" == gaea-c6 ) then
+   module reset
+
 else if ( "$L_MACHINE" == derecho ) then
    module purge
 

etc/lmod-setup.sh

@@ -47,6 +47,9 @@ elif [ "$L_MACHINE" = singularity ]; then
 elif [ "$L_MACHINE" = gaea ]; then
    module reset 
 
+elif [ "$L_MACHINE" = gaea-c6 ]; then
+   module reset 
+
 elif [ "$L_MACHINE" = derecho ]; then
    module purge
 

modulefiles/build_gaea-c6_intel.lua

@@ -0,0 +1,35 @@
+help([[
+This module loads libraries for building the UFS SRW App on
+the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0
+]])
+
+whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C6 ]===])
+
+prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+load(pathJoin("stack-python", stack_python_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("srw_common")
+
+unload("darshan-runtime/3.4.4")
+unload("cray-pmi/6.1.13")
+
+setenv("CFLAGS","-diag-disable=10441")
+setenv("FFLAGS","-diag-disable=10441")
+
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Platform","gaea-c6.intel")

modulefiles/tasks/gaea-c6/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/gaea-c6/plot_allvars.local.lua

@@ -0,0 +1,4 @@
+unload("python")
+load("conda")
+
+setenv("SRW_GRAPHICS_ENV", "srw_graphics")

modulefiles/tasks/gaea-c6/python_srw.lua

@@ -0,0 +1,8 @@
+load("darshan-runtime/3.4.4")
+unload("python")
+load("conda")
+
+setenv("SRW_ENV", "srw_app")
+setenv("LD_PRELOAD", "/usr/lib64/libstdc++.so.6")
+setenv("FI_VERBS_PREFER_XRC", "0")
+

modulefiles/tasks/gaea-c6/run_vx.local.lua

@@ -0,0 +1,26 @@
+--[[
+Compiler-specific modules are used for met and metplus libraries
+--]]
+local met_ver = (os.getenv("met_ver") or "11.1.0")
+local metplus_ver = (os.getenv("metplus_ver") or "5.1.0")
+if (mode() == "load") then
+  load(pathJoin("met", met_ver))
+  load(pathJoin("metplus",metplus_ver))
+end
+local base_met = os.getenv("met_ROOT") or os.getenv("MET_ROOT")
+local base_metplus = os.getenv("metplus_ROOT") or os.getenv("METPLUS_ROOT")
+
+setenv("MET_INSTALL_DIR", base_met)
+setenv("MET_BIN_EXEC",    pathJoin(base_met,"bin"))
+setenv("MET_BASE",        pathJoin(base_met,"share/met"))
+setenv("MET_VERSION",     met_ver)
+setenv("METPLUS_VERSION", metplus_ver)
+setenv("METPLUS_ROOT",    base_metplus)
+setenv("METPLUS_PATH",    base_metplus)
+
+if (mode() == "unload") then
+  unload(pathJoin("met", met_ver))
+  unload(pathJoin("metplus",metplus_ver))
+end
+load("conda")
+setenv("SRW_ENV", "srw_app")

modulefiles/tasks/gaea/run_vx.local.lua

@@ -24,4 +24,3 @@ if (mode() == "unload") then
 end
 load("conda")
 setenv("SRW_ENV", "srw_app")
-setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6")

modulefiles/wflow_gaea-c6.lua

@@ -0,0 +1,19 @@
+help([[
+This module loads python environement for running the UFS SRW App on
+the NOAA RDHPC machine Gaea C6
+]])
+
+whatis([===[Loads libraries needed for running the UFS SRW App on gaea c6 ]===])
+
+unload("python")
+prepend_path("MODULEPATH","/ncrc/proj/epic/rocoto/modulefiles/")
+load("rocoto")
+load("conda")
+
+pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.2.0/")
+
+if mode() == "load" then
+   LmodMsgRaw([===[Please do the following to activate conda:
+       > conda activate srw_app
+]===])
+end

modulefiles/wflow_gaea.lua

@@ -3,14 +3,14 @@ This module loads python environement for running the UFS SRW App on
 the NOAA RDHPC machine Gaea C5
 ]])
 
-whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===])
+whatis([===[Loads libraries needed for running the UFS SRW App on gaea c5 ]===])
 
 unload("python")
 prepend_path("MODULEPATH","/ncrc/proj/epic/rocoto/modulefiles/")
 load("rocoto")
 load("conda")
 
-pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.1.0/")
+pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.2.0/")
 
 if mode() == "load" then
    LmodMsgRaw([===[Please do the following to activate conda:

tests/WE2E/machine_suites/comprehensive.gaea-c6

@@ -0,0 +1 @@
+comprehensive.orion

tests/WE2E/machine_suites/coverage.gaea-c6

@@ -0,0 +1,11 @@
+community
+custom_ESGgrid_NewZealand_3km
+grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
+grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
+grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km
+grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR
+grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
+grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot
+2020_CAPE
+2020_easter_storm

tests/WE2E/setup_WE2E_tests.sh

@@ -45,7 +45,7 @@ function usage {
 
 }
 
-machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud )
+machines=( hera jet cheyenne derecho orion wcoss2 gaea gaea-c6 odin singularity macos noaacloud )
 
 if [ "$1" = "-h" ] ; then usage ; fi
 [[ $# -le 2 ]] && usage

tests/build.sh

@@ -21,7 +21,7 @@ function usage() {
   exit 1
 }
 
-machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud )
+machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea gaea-c6 odin singularity macos noaacloud )
 
 [[ $# -gt 4 ]] && usage
 

ush/machine/gaea-c6.yaml

@@ -0,0 +1,56 @@
+platform:
+  WORKFLOW_MANAGER: rocoto
+  NCORES_PER_NODE: 128
+  SCHED: slurm
+  WE2E_TEST_DATA: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop
+  TEST_CCPA_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_NOHRSC_OBS_DIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/obs_data/nohrsc/proc
+  DOMAIN_PREGEN_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/FV3LAM_pregen
+  QUEUE_DEFAULT: normal
+  QUEUE_FCST: normal
+  QUEUE_HPSS: normal
+  REMOVE_MEMORY: True
+  PARTITION_HPSS: eslogin_c6
+  RUN_CMD_FCST: srun --export=ALL
+  RUN_CMD_POST: srun --export=ALL -n $nprocs
+  RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs
+  RUN_CMD_SERIAL: time
+  RUN_CMD_UTILS: srun --export=ALL -n $nprocs
+  SCHED_NATIVE_CMD: --clusters=c6 --partition=batch --export=NONE
+  SCHED_NATIVE_CMD_HPSS: --clusters=es --partition=eslogin_c6 --export=NONE
+  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
+  TEST_EXTRN_MDL_SOURCE_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data
+  TEST_PREGEN_BASEDIR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/FV3LAM_pregen
+  TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir
+  TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir
+  TEST_VX_FCST_INPUT_BASEDIR: '{{ "/gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}'
+  FIXaer: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_aer
+  FIXgsi: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_gsi
+  FIXgsm: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_am
+  FIXlut: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_lut
+  FIXorg: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_orog
+  FIXsfc: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/fix/fix_sfc_climo
+  FIXshp: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/NaturalEarth
+  EXTRN_MDL_DATA_STORES: aws
+data:
+  ics_lbcs:
+    FV3GFS:
+      nemsio: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh}
+      grib2: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} 
+      netcdf: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh} 
+    RAP: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
+    HRRR: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
+    RAP: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
+    GSMGFS: /gpfs/f6/bil-fire8/world-shared/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh}
+rocoto:
+  tasks:
+    metatask_run_ensemble:
+      task_run_fcst_mem#mem#:
+        cores:
+        native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}'
+        nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}'
+        nnodes:
+        nodesize:
+        ppn: