RRFS fire weather grid: post processing and product generation tasks

parm/FV3LAM_wflow_nonDA.xml

@@ -36,6 +36,7 @@ Workflow task names.
 <!ENTITY MAKE_LBCS_TN      "{{ make_lbcs_tn }}">
 <!ENTITY RUN_FCST_TN       "{{ run_fcst_tn }}">
 <!ENTITY RUN_POST_TN       "{{ run_post_tn }}">
+<!ENTITY RUN_PRDGEN_TN     "{{ run_prdgen_tn }}">
 
 <!ENTITY TAG                  "{{ tag }}">
 <!ENTITY NET                  "{{ net }}">
@@ -47,6 +48,11 @@ Flags that specify whether to run the preprocessing tasks.
 <!ENTITY RUN_TASK_MAKE_OROG      "{{ run_task_make_orog | upper }}">
 <!ENTITY RUN_TASK_MAKE_SFC_CLIMO "{{ run_task_make_sfc_climo | upper }}">
 
+<!--
+Flag that specifies whether to run the product generation task.
+-->
+<!ENTITY RUN_TASK_RUN_PRDGEN "{{ run_task_run_prdgen | upper }}">
+
 <!--
 Number of physical cores per node for the current machine.  This is used
 below in the <nodesize> tag, but that tag is not clearly documented.  This
@@ -106,14 +112,16 @@ define resources used for each tasks
 <!ENTITY RESOURCES_MAKE_LBCS "<nodes>{{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }}:tpp={{ tpp_make_lbcs }}</nodes>">
 <!ENTITY RESOURCES_RUN_FCST "<nodes>{{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ tpp_run_fcst }}</nodes>">
 <!ENTITY RESOURCES_RUN_POST "<nodes>{{ nnodes_run_post }}:ppn={{ ppn_run_post }}:tpp={{ tpp_run_post }}</nodes>">
+<!ENTITY RESOURCES_RUN_PRDGEN "<nodes>{{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }}</nodes>">
 <!ENTITY NODESIZE_ALL    "">
-<!ENTITY NATIVE_ALL    "<native>-l place=excl</native>">
+<!ENTITY NATIVE_ALL    "<native>-l place=scatter</native>">
 <!ENTITY NATIVE_RUN_FCST    "<native>-l place=excl</native>">
 {%- elif machine in ["JET"] %}
 <!ENTITY RESOURCES_MAKE_ICS "<nodes>{{ nnodes_make_ics }}:ppn={{ ppn_make_ics }}</nodes>">
 <!ENTITY RESOURCES_MAKE_LBCS "<nodes>{{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }}</nodes>">
 <!ENTITY RESOURCES_RUN_FCST "<cores>{{ ncores_run_fcst }}</cores>">
 <!ENTITY RESOURCES_RUN_POST "<nodes>{{ nnodes_run_post }}:ppn={{ ppn_run_post }}</nodes>">
+<!ENTITY RESOURCES_RUN_PRDGEN "<nodes>{{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }}</nodes>">
 <!ENTITY NODESIZE_ALL    "<nodesize>{{ ncores_per_node }}</nodesize>">
 <!ENTITY NATIVE_ALL    "">
 <!ENTITY NATIVE_RUN_FCST    "<native>{{ native_run_fcst }} &RRFS_RESERVE;</native>">
@@ -122,6 +130,7 @@ define resources used for each tasks
 <!ENTITY RESOURCES_MAKE_LBCS "<nodes>{{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }}</nodes>">
 <!ENTITY RESOURCES_RUN_FCST "<nodes>{{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}</nodes>">
 <!ENTITY RESOURCES_RUN_POST "<nodes>{{ nnodes_run_post }}:ppn={{ ppn_run_post }}</nodes>">
+<!ENTITY RESOURCES_RUN_PRDGEN "<nodes>{{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }}</nodes>">
 <!ENTITY NODESIZE_ALL    "<nodesize>{{ ncores_per_node }}</nodesize>">
 <!ENTITY NATIVE_ALL    "">
 <!ENTITY NATIVE_RUN_FCST    "">
@@ -130,6 +139,7 @@ define resources used for each tasks
 <!ENTITY RESOURCES_MAKE_LBCS "<nodes>{{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }}</nodes>">
 <!ENTITY RESOURCES_RUN_FCST "<nodes>{{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}</nodes>">
 <!ENTITY RESOURCES_RUN_POST "<nodes>{{ nnodes_run_post }}:ppn={{ ppn_run_post }}</nodes>">
+<!ENTITY RESOURCES_RUN_PRDGEN "<nodes>{{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }}</nodes>">
 <!ENTITY NODESIZE_ALL    "<nodesize>{{ ncores_per_node }}</nodesize>">
 <!ENTITY NATIVE_ALL    "">
 <!ENTITY NATIVE_RUN_FCST    "">
@@ -482,4 +492,38 @@ MODULES_RUN_TASK_FP script.
       </task>
   </metatask>
 
+{%- if run_task_run_prdgen %}
+<!--
+************************************************************************
+************************************************************************
+-->
+  <metatask name="&RUN_PRDGEN_TN;">
+    <var name="fhr">{% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %}</var>
+      <task name="&RUN_PRDGEN_TN;{{ uscore_ensmem_name }}_f#fhr#" cycledefs="forecast" maxtries="{{ maxtries_run_prdgen }}">
+        &RSRV_DEFAULT;
+        <command>&LOAD_MODULES_RUN_TASK_FP; "&RUN_PRDGEN_TN;" "&JOBSdir;/JRRFS_RUN_PRDGEN"</command>
+	&RESOURCES_RUN_PRDGEN;
+        &NATIVE_ALL;
+        <walltime>{{ wtime_run_prdgen }}</walltime>
+        &NODESIZE_ALL;
+        <jobname>&RUN_PRDGEN_TN;_f#fhr#</jobname>
+        <join><cyclestr>&LOGDIR;/&RUN_PRDGEN_TN;_f#fhr#_@Y@m@[email protected]</cyclestr></join>
+
+        <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>
+        <envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+        <envar><name>CDATE</name><value><cyclestr>@Y@m@d@H</cyclestr></value></envar>
+        <envar><name>CYCLE_DIR</name><value><cyclestr>&CYCLE_BASEDIR;/@Y@m@d@H</cyclestr></value></envar>
+        <envar><name>SLASH_ENSMEM_SUBDIR</name><value><cyclestr>{{ slash_ensmem_subdir }}</cyclestr></value></envar>
+        <envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+        <envar><name>fhr</name><value>#fhr#</value></envar>
+        <envar><name>TMMARK</name><value>tm00</value></envar>
+
+        <dependency>
+          <taskdep task="&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#"/>
+        </dependency>
+
+      </task>
+  </metatask>
+{%- endif %}
+
 </workflow>

scripts/exrrfs_run_post.sh

@@ -345,6 +345,11 @@ if [ -f IFIFIP.GrbF${post_fhr} ]; then
   wgrib2 IFIFIP.GrbF${post_fhr} -set center 7 -grib ${bgifi} >>$pgmout 2>>errfile
 fi
 
+# Keep latlons_corners.txt file for RRFS fire weather grid
+if [ ${PREDEF_GRID_NAME} = "RRFS_FIREWX_1.5km" ]; then
+  cp ${postprd_dir}/${fhr}/latlons_corners.txt.f${fhr} ${postprd_dir}
+fi
+
 #
 #-----------------------------------------------------------------------
 #   clean forecast netcdf files for saving space

scripts/exrrfs_run_prdgen.sh

@@ -90,7 +90,7 @@ case $MACHINE in
     ;;
 
   "HERA")
-    APRUN="srun"
+    APRUN="srun --export=ALL"
     ;;
 
   "ORION")
@@ -226,14 +226,19 @@ cp ${postprd_dir}/${natlev} ${COMOUT}/${natlev}
 if [ -f  ${postprd_dir}/${ififip} ]; then
   cp ${postprd_dir}/${ififip} ${COMOUT}/${ififip}
 fi
-cp ${postprd_dir}/${testbed}  ${COMOUT}/${testbed}
+if [ -f  ${postprd_dir}/${testbed} ]; then
+  cp ${postprd_dir}/${testbed}  ${COMOUT}/${testbed}
+fi
 
 wgrib2 ${COMOUT}/${prslev} -s > ${COMOUT}/${prslev}.idx
 wgrib2 ${COMOUT}/${natlev} -s > ${COMOUT}/${natlev}.idx
 if [ -f ${COMOUT}/${ififip} ]; then
   wgrib2 ${COMOUT}/${ififip} -s > ${COMOUT}/${ififip}.idx
 fi
-wgrib2 ${COMOUT}/${testbed} -s > ${COMOUT}/${testbed}.idx
+if [ -f ${COMOUT}/${testbed} ]; then
+  wgrib2 ${COMOUT}/${testbed} -s > ${COMOUT}/${testbed}.idx
+fi
+
 # Remap to additional output grids if requested
 
 if [ "${DO_PARALLEL_PRDGEN}" = "TRUE" ]; then
@@ -320,6 +325,39 @@ if [ "${DO_PARALLEL_PRDGEN}" = "TRUE" ]; then
   rm -fr $DATAprdgen
   rm -f $DATA/*.t${cyc}z.*.f${fhr}.*.grib2
 
+elif [ ${PREDEF_GRID_NAME} = "RRFS_FIREWX_1.5km" ]; then
+  #
+  # Processing for the RRFS fire weather grid
+  #
+  DATA=${postprd_dir}/${fhr}
+  cd $DATA
+
+  # set GTYPE=2 for GRIB2
+  GTYPE=2
+
+  cat > itagfw <<EOF
+CONUS
+$GTYPE
+EOF
+
+  # Read in corner lat lons from UPP text file
+  export FORT11=${postprd_dir}/latlons_corners.txt.f${fhr}
+  export FORT45=itagfw
+
+  # Calculate the wgrib2 gridspecs for the fire weather grid
+  $APRUN $EXECdir/firewx_gridspecs.exe >> $pgmout 2>errfile
+  export err=$?; err_chk
+
+  grid_specs_firewx=`head $DATA/copygb_gridnavfw.txt`
+  eval infile=${postprd_dir}/${net4}.t${cyc}z.prslev.f${fhr}.grib2
+
+  wgrib2 ${infile} -set_bitmap 1 -set_grib_type c3 -new_grid_winds grid \
+   -new_grid_vectors "UGRD:VGRD:USTM:VSTM:VUCSH:VVCSH" \
+   -new_grid_interpolation neighbor \
+   -if ":(WEASD|APCP|NCPCP|ACPCP|SNOD):" -new_grid_interpolation budget -fi \
+   -new_grid ${grid_specs_firewx} ${COMOUT}/rrfs.t${cyc}z.prslev.f${fhr}.firewx.grib2
+  wgrib2 ${COMOUT}/rrfs.t${cyc}z.prslev.f${fhr}.firewx.grib2 -s > ${COMOUT}/rrfs.t${cyc}z.prslev.f${fhr}.firewx.grib2.idx
+
 else
   #
   # use single core to process all addition grids.

sorc/Externals.cfg

@@ -48,7 +48,7 @@ protocol = git
 repo_url = https://github.com/NOAA-GSL/rrfs_utl
 # Specify either a branch name or a hash but not both.
 # branch = develop
-hash = 83d6805
+hash = 0fce0fa
 local_path = rrfs_utl
 required = True
 

ush/config_defaults.sh

@@ -1507,6 +1507,9 @@ SAVE_CYCLE_LOG="TRUE"
 # SFC_CLIMO_DIR:
 # Same as GRID_DIR but for the surface climatology generation task.
 #
+# RUN_TASK_RUN_PRDGEN:
+# Same as RUN_TASK_MAKE_GRID but for the product generation task.
+#
 # IS_RTMA:
 # If true, some ICs,LBCs,GSI rocoto tasks will be turned off
 #
@@ -1535,6 +1538,8 @@ OROG_DIR=""
 RUN_TASK_MAKE_SFC_CLIMO="FALSE"
 SFC_CLIMO_DIR=""
 
+RUN_TASK_RUN_PRDGEN="TRUE"
+
 #
 NCORES_PER_NODE=24 #Jet default value
 IS_RTMA="FALSE"

ush/generate_FV3LAM_wflow.sh

@@ -400,6 +400,8 @@ settings="\
   'run_task_make_grid': ${RUN_TASK_MAKE_GRID}
   'run_task_make_orog': ${RUN_TASK_MAKE_OROG}
   'run_task_make_sfc_climo': ${RUN_TASK_MAKE_SFC_CLIMO}
+#
+  'run_task_run_prdgen': ${RUN_TASK_RUN_PRDGEN}
 #
   'is_rtma':  ${IS_RTMA}
   'fg_rootdir': ${FG_ROOTDIR}

ush/sample_configs/RRFSFW/config.rrfsfw.hera.sh

@@ -0,0 +1,72 @@
+MACHINE="hera"
+ACCOUNT="fv3-cam"
+HPSS_ACCOUNT="fv3-cam"
+EXPT_SUBDIR="test_firewx"
+
+envir="rrfsfw"
+NET="rrfsfw"
+TAG="c0v00"
+MODEL="rrfsfw"
+RUN="rrfsfw"
+
+STMP="/scratch2/NCEPDEV/stmp3/Benjamin.Blake/test_firewx"
+PTMP="/scratch2/NCEPDEV/stmp3/Benjamin.Blake/test_firewx"
+
+VERBOSE="TRUE"
+PRINT_ESMF="TRUE"
+
+USE_CRON_TO_RELAUNCH="FALSE"
+CRON_RELAUNCH_INTVL_MNTS="03"
+
+PREEXISTING_DIR_METHOD="rename"
+
+PREDEF_GRID_NAME="RRFS_FIREWX_1.5km"
+#CENTLON="-106.0"
+#CENTLAT="39.2"
+QUILTING="TRUE"
+
+CCPP_PHYS_SUITE="FV3_HRRR_gf_nogwd"
+FCST_LEN_HRS="36"
+LBC_SPEC_INTVL_HRS="1"
+BOUNDARY_LEN_HRS="36"
+BOUNDARY_PROC_GROUP_NUM="36"
+
+DATE_FIRST_CYCL="20231111"
+DATE_LAST_CYCL="20231111"
+CYCL_HRS=( "18" )
+
+RUN_TASK_MAKE_GRID="TRUE"
+RUN_TASK_MAKE_OROG="TRUE"
+RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="TRUE"
+NNODES_MAKE_SFC_CLIMO="6"
+PPN_MAKE_SFC_CLIMO="12"
+
+# Temporary solution until high resolution fix files are added to common directory
+SFC_CLIMO_INPUT_DIR="/scratch2/NCEPDEV/fv3-cam/Benjamin.Blake/fix_sfc_climo"
+
+EXTRN_MDL_NAME_ICS="RRFS"
+EXTRN_MDL_NAME_LBCS="RRFS"
+
+PPN_MAKE_ICS="6"
+NNODES_MAKE_ICS="8"
+PPN_MAKE_LBCS="36"
+NNODES_MAKE_LBCS="1"
+TPP_MAKE_LBCS="1"
+
+WTIME_RUN_FCST="01:10:00"
+PPN_RUN_FCST="14"
+NNODES_RUN_FCST="18"
+TPP_RUN_FCST="2"
+
+DO_NON_DA_RUN="TRUE"
+DO_RETRO="TRUE"
+DO_SMOKE_DUST="FALSE"
+VCOORD_FILE="global_hyblev_fcst_rrfsL65.txt"
+WFLOW_XML_TMPL_FN="FV3LAM_wflow_nonDA.xml"
+FV3_NML_YAML_CONFIG_FN=""
+FV3_NML_BASE_SUITE_FN="input.nml.RRFSFW"
+
+USE_USER_STAGED_EXTRN_FILES="TRUE"
+EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/NCEPDEV/fv3-cam/Benjamin.Blake/input_model_data/RRFS/2023111118"
+EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch2/NCEPDEV/fv3-cam/Benjamin.Blake/input_model_data/RRFS/2023111118"

ush/sample_configs/RRFSFW/config.sh_rrfsfw → ush/sample_configs/RRFSFW/config.rrfsfw.wcoss2.sh

@@ -3,11 +3,11 @@ ACCOUNT="RRFS-DEV"
 HPSS_ACCOUNT="RRFS-DEV"
 EXPT_SUBDIR="test_firewx"
 
-envir="test"
-NET="test"
+envir="rrfsfw"
+NET="rrfsfw"
 TAG="c0v00"
-MODEL="test"
-RUN="test"
+MODEL="rrfsfw"
+RUN="rrfsfw"
 
 STMP=""
 PTMP=""
@@ -38,6 +38,7 @@ CYCL_HRS=( "18" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="TRUE"
 NNODES_MAKE_SFC_CLIMO="6"
 PPN_MAKE_SFC_CLIMO="12"
 

ush/sample_configs/non-DA_eng/config.nonDA.grib2.hera.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "00" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "00" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.grib2.jet.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "00" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.grib2.orion.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "00" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hera.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "06" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "06" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.netcdf.jet.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "06" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.netcdf.orion.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "06" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"

ush/sample_configs/non-DA_eng/config.nonDA.netcdf.wcoss2.sh

@@ -34,6 +34,7 @@ CYCL_HRS=( "06" )
 RUN_TASK_MAKE_GRID="TRUE"
 RUN_TASK_MAKE_OROG="TRUE"
 RUN_TASK_MAKE_SFC_CLIMO="TRUE"
+RUN_TASK_RUN_PRDGEN="FALSE"
 
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"