From 5c14d30a47780392c8f8d327d186b74d73c5fa70 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:36:38 -0500 Subject: [PATCH 01/25] Build with container spack-stack-1.8.0 --- modulefiles/module_base.container.lua | 57 +++++++++++++++++++++++++++ run.container.ver | 36 +++++++++++++++++ versions/build.container.ver | 10 +++++ versions/run.container.ver | 43 ++++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 modulefiles/module_base.container.lua create mode 100644 run.container.ver create mode 100644 versions/build.container.ver create mode 100644 versions/run.container.ver diff --git a/modulefiles/module_base.container.lua b/modulefiles/module_base.container.lua new file mode 100644 index 0000000000..72b11a2b4e --- /dev/null +++ b/modulefiles/module_base.container.lua @@ -0,0 +1,57 @@ +help([[ +Load environment to run GFS on Hera +]]) + +local spack_mod_path=(os.getenv("spack_mod_path") or "None") +prepend_path("MODULEPATH", spack_mod_path) + +setenv("CC", "icc" ) +setenv("CXX","icpc" ) +setenv("F77","ifort" ) +setenv("F90","ifort" ) +setenv("FC", "ifort" ) +load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) +load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) +--load(pathJoin("python", (os.getenv("python_ver") or "None"))) + +--load(pathJoin("hpss", (os.getenv("hpss_ver") or "None"))) +--load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) +--load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) +load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) +load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) +load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) +--load(pathJoin("R", (os.getenv("R_ver") or "None"))) +--load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) + +load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) +load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) +load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) + +load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) +load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) +load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) +load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) +load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) +load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) +load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) +load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) +load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) +load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) +load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) +load(pathJoin("met", (os.getenv("met_ver") or "None"))) +load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) +load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) + +--setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +--setenv("WGRIB","wgrib") +--setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) + +whatis("Description: GFS run environment") diff --git a/run.container.ver b/run.container.ver new file mode 100644 index 0000000000..4f2958f640 --- /dev/null +++ b/run.container.ver @@ -0,0 +1,36 @@ +export spack_stack_ver=1.6.0 +export python_ver=3.11.6 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 diff --git a/versions/build.container.ver b/versions/build.container.ver new file mode 100644 index 0000000000..7a2b50464b --- /dev/null +++ b/versions/build.container.ver @@ -0,0 +1,10 @@ +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.12.1 +export spack_env=global-workflow-env +source "${HOMEgfs:-}/versions/run.container.ver" +export spack_mod_path="/opt/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +export I_MPI_CC=$CC +export I_MPI_CXX=$CXX +export I_MPI_F77=$FC +export I_MPI_FC=$FC +export I_MPI_F90=$FC diff --git a/versions/run.container.ver b/versions/run.container.ver new file mode 100644 index 0000000000..a2a66d5041 --- /dev/null +++ b/versions/run.container.ver @@ -0,0 +1,43 @@ +export spack_stack_ver=1.6.0 +export python_ver=3.11.6 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 +export esmf_ver=8.6.1 +export fms_ver=2024.02 +export mapl_ver=2.46.3-esmf-8.6.1 +export gftl_shared_ver=1.9.0 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export g2_ver=3.5.1 +export cmake_ver=3.27.9 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export parallelio_ver=2.6.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 From b10005715ded7bae2f308de44c868a424d8b8995 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:41:51 -0500 Subject: [PATCH 02/25] Build with container spack-stack-1.8.0 --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index bd8f13d867..99ecd2bb92 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 +Subproject commit 99ecd2bb9267e0365020334746a78d1e59edde61 From 1fe4c1567ba1a168819fbcc7184f41764868f041 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:44:05 -0500 Subject: [PATCH 03/25] Build with container spack-stack-1.8.0 --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 529bb796be..7cfe53aeda 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d +Subproject commit 7cfe53aeda97858d67726ace1ab77d593049362f From 29782a23d3f092f2c26d106055aac96dc466b447 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:45:43 -0500 Subject: [PATCH 04/25] Build with container spack-stack-1.8.0 --- sorc/gsi_monitor.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index e1f9f21af1..6650ff272f 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 +Subproject commit 6650ff272f725ad2974a3528eea84bbd32922c60 From ddb333ed452705f92549027e36df4f61656fd197 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:46:01 -0500 Subject: [PATCH 05/25] Build with container spack-stack-1.8.0 --- sorc/link_workflow.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 92404afc01..13b44b902b 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -77,6 +77,8 @@ case "${machine}" in "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; + "frontera") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;; + "container") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" exit 1 From 5acd7ee6452a5dd7a19e41bac0631b64b177bc99 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:47:26 -0500 Subject: [PATCH 06/25] Build with container spack-stack-1.8.0 --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 9382fd01c2..07f7b5fb82 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec +Subproject commit 07f7b5fb828db2cdfa5d9d5b192a4ab59e74608b From 2ca21b9ae69f32254f6e970ae2283629b8fdbd99 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:48:13 -0500 Subject: [PATCH 07/25] Build with container spack-stack-1.8.0 --- sorc/ufs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 3ef2e6bd72..9b897bc355 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 +Subproject commit 9b897bc3551638125c945831c66d817924e63824 From 20b765d8c86aa972de1b520cb0eb92fe0fa8c30f Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 12 Sep 2024 10:53:47 -0500 Subject: [PATCH 08/25] Build with container spack-stack-1.8.0 --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index fcc9f8461d..ca753084e6 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit fcc9f8461db5eafbfd1f080da61ea79156ca0145 +Subproject commit ca753084e6e3111cccaf5e25e5e3c791743c5447 From 8a8305b7fcf779275fad3560ed83b72f6d5d063d Mon Sep 17 00:00:00 2001 From: benjamin-cash <62720073+benjamin-cash@users.noreply.github.com> Date: Thu, 12 Sep 2024 12:50:41 -0400 Subject: [PATCH 09/25] Update .gitmodules Point to my forks so updates can be accepted --- .gitmodules | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5c9e569243..812cdd2bff 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,28 +1,28 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/benjamin-cash/ufs-weather-model ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + url = https://github.com/benjamin-cash/gfs-utils [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS.git + url = https://github.com/benjamin-cash/UFS_UTILS.git [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd - url = https://github.com/NOAA-EMC/EMC_verif-global.git + url = https://github.com/benjamin-cash/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd - url = https://github.com/NOAA-EMC/GSI.git + url = https://github.com/benjamin-cash/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git [submodule "sorc/gsi_utils.fd"] path = sorc/gsi_utils.fd - url = https://github.com/NOAA-EMC/GSI-Utils.git + url = https://github.com/benjamin-cash/GSI-Utils.git [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd - url = https://github.com/NOAA-EMC/GSI-Monitor.git + url = https://github.com/benjamin-cash/GSI-Monitor.git From 55b38b6b51c7f997f697965b51fdd7889bec01da Mon Sep 17 00:00:00 2001 From: benjamin-cash <62720073+benjamin-cash@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:58:19 -0400 Subject: [PATCH 10/25] Update .gitmodules Reverting change to EMC_verif --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 812cdd2bff..a241d86d29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,7 +13,7 @@ url = https://github.com/benjamin-cash/UFS_UTILS.git [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd - url = https://github.com/benjamin-cash/EMC_verif-global.git + url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd url = https://github.com/benjamin-cash/GSI.git From 9f98cb2094f1c7417f5ccb3d630f4008abfda5aa Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Fri, 13 Sep 2024 09:59:15 -0500 Subject: [PATCH 11/25] Update to identify container --- ush/detect_machine.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a6040e..2e7c7287a7 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -38,7 +38,8 @@ case $(hostname -f) in [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4 login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 - + *frontera.tacc.utexas.edu) MACHINE_ID=frontera ;; + *orc.gmu.edu) MACHINE_ID=hopper ;; login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 @@ -61,7 +62,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then fi # Try searching based on paths since hostname may not match on compute nodes -if [[ -d /lfs/h3 ]]; then +if [[ -d /opt/container-scripts ]]; then + # We are in the container + MACHINE_ID=container +elif [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then From bb08d9f620abb97dedd2c6644f4ddf8f84b2dd11 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Fri, 13 Sep 2024 11:05:12 -0500 Subject: [PATCH 12/25] Update module setup for container --- ush/load_fv3gfs_modules.sh | 2 +- ush/module-setup.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index ff6f64cece..84fb1f8f26 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud") + "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud" | "container") module load "module_base.${MACHINE_ID}" ;; *) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 398562652d..80e1841013 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -3,7 +3,11 @@ set -u source "${HOMEgfs}/ush/detect_machine.sh" -if [[ ${MACHINE_ID} = jet* ]] ; then +if [[ ${MACHINE_ID} = container ]] ; then + source /usr/lmod/lmod/init/bash + module purge + +elif [[ ${MACHINE_ID} = jet* ]] ; then # We are on NOAA Jet if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash From 4c3490bdeb7a738cadb036503ebcb10c0adbf667 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Wed, 2 Oct 2024 14:36:21 -0500 Subject: [PATCH 13/25] ufs_model.fd submodule updates --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index ca753084e6..d8324b1ecb 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit ca753084e6e3111cccaf5e25e5e3c791743c5447 +Subproject commit d8324b1ecb8755a465a0b2189021ecdc4118f1fa From 24d2f9237ddf44e69229fdbdf41b9f6fe8bb34c8 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Wed, 2 Oct 2024 15:04:38 -0500 Subject: [PATCH 14/25] More submodule updates --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index d8324b1ecb..444a4b38c1 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit d8324b1ecb8755a465a0b2189021ecdc4118f1fa +Subproject commit 444a4b38c191d84092c56989e619c2a334fa8040 From 1ebec5c4f8254126e8f4f78ae9f6df20d6c6597d Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Wed, 2 Oct 2024 15:51:41 -0500 Subject: [PATCH 15/25] Pushing submodule updates up the tree --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 444a4b38c1..f2c0585ed4 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 444a4b38c191d84092c56989e619c2a334fa8040 +Subproject commit f2c0585ed4bf686944cf5fae66f790dbae1dc692 From 173d03f5ca9f1b568fbe67bf3caf332e7af8231e Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 09:56:19 -0500 Subject: [PATCH 16/25] Updating hashes --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index f2c0585ed4..30b9d26b5a 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit f2c0585ed4bf686944cf5fae66f790dbae1dc692 +Subproject commit 30b9d26b5abb7f5003911a33156840760ca80056 From 9e3222aecbe4f486ced3babc52d1733af2afae3f Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 12:12:41 -0400 Subject: [PATCH 17/25] Add minimal module file for Hopper --- modulefiles/module_base.hopper.lua | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 modulefiles/module_base.hopper.lua diff --git a/modulefiles/module_base.hopper.lua b/modulefiles/module_base.hopper.lua new file mode 100644 index 0000000000..de1f533513 --- /dev/null +++ b/modulefiles/module_base.hopper.lua @@ -0,0 +1,8 @@ +help([[ +Minimal environment to run GFS on Hopper via containers +]]) + +load("hosts/hopper") +load("OneAPI/2022.1.2") +load("singularity") +load("mpi/2021.5.1") From b2389470395773274f03a39ed2dd6e3e32e0c475 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 13:56:34 -0500 Subject: [PATCH 18/25] Update branch names --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 30b9d26b5a..301334a762 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 30b9d26b5abb7f5003911a33156840760ca80056 +Subproject commit 301334a762480c7c3047bd0a7ff07105bc8166b1 From 4a8168121348de6d112012f85689c4a5f9519fcd Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 14:02:31 -0500 Subject: [PATCH 19/25] Updating branch names --- sorc/ufs_model.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 301334a762..dde7bf7532 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 301334a762480c7c3047bd0a7ff07105bc8166b1 +Subproject commit dde7bf75326a15b72ed6a74ad40249ea70647cb2 From 1c86386ed84022f870efbadef87a9e16b85cc613 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 14:14:00 -0500 Subject: [PATCH 20/25] Minimal module setup for Frontera and containers --- modulefiles/module_base.frontera.lua | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 modulefiles/module_base.frontera.lua diff --git a/modulefiles/module_base.frontera.lua b/modulefiles/module_base.frontera.lua new file mode 100644 index 0000000000..07447e64de --- /dev/null +++ b/modulefiles/module_base.frontera.lua @@ -0,0 +1,6 @@ +help([[ +Minimal environment to run GFS on Frontera via containers +]]) +load("intel/23.1.0") +load("impi/21.9.0") +load("tacc-apptainer") From eb3e39b0cc3e315ab23558e69acb5d480e2ed025 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 14:28:11 -0500 Subject: [PATCH 21/25] Add hopper and frontera to machine list --- ush/load_fv3gfs_modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 84fb1f8f26..eb7bc959f7 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud" | "container") + "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud" | "container" | "frontera" | "hopper") module load "module_base.${MACHINE_ID}" ;; *) From cfdbb52d507b8313c5832e63d9257a3a1f705888 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 14:36:58 -0500 Subject: [PATCH 22/25] Update spack versions --- versions/spack.ver | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 versions/spack.ver diff --git a/versions/spack.ver b/versions/spack.ver new file mode 100644 index 0000000000..c70c95f3ae --- /dev/null +++ b/versions/spack.ver @@ -0,0 +1,43 @@ +export spack_stack_ver=1.8.0 +export python_ver=3.11.6 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 +export esmf_ver=8.6.1 +export fms_ver=2024.02 +export mapl_ver=2.46.3-esmf-8.6.1 +export gftl_shared_ver=1.9.0 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export g2_ver=3.5.1 +export cmake_ver=3.27.9 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export parallelio_ver=2.6.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 From 9c638290ca01bc3ca7662f808ed1a70acc4c6234 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 16:56:25 -0400 Subject: [PATCH 23/25] Add intel and mpi version numbers --- versions/spack.ver | 2 ++ 1 file changed, 2 insertions(+) diff --git a/versions/spack.ver b/versions/spack.ver index c70c95f3ae..cbe2f9b3ac 100644 --- a/versions/spack.ver +++ b/versions/spack.ver @@ -1,5 +1,7 @@ export spack_stack_ver=1.8.0 export python_ver=3.11.6 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.12.1 export jasper_ver=2.0.32 export libpng_ver=1.6.37 From 19192131b734125c3509c9d0cc49af057ad36216 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Thu, 3 Oct 2024 16:59:38 -0400 Subject: [PATCH 24/25] Update fix file directories --- sorc/link_workflow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 13b44b902b..b92ce56198 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -78,7 +78,8 @@ case "${machine}" in "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; "frontera") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;; - "container") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;; + "container") FIX_DIR="/groups/BCASH/ufs_input/fix" ;; + "hopper") FIX_DIR="/groups/BCASH/ufs_input/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" exit 1 From c36aeb17046071504d17ca118026bad9cfdb97d8 Mon Sep 17 00:00:00 2001 From: Benjamin Cash Date: Fri, 4 Oct 2024 10:53:54 -0400 Subject: [PATCH 25/25] Update hosts to include hopper and frontera --- workflow/hosts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/hosts.py b/workflow/hosts.py index 34ea067ade..a4c590f2cb 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -17,7 +17,8 @@ class Host: SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'GAEA', - 'AWSPW', 'AZUREPW', 'GOOGLEPW'] + 'AWSPW', 'AZUREPW', 'GOOGLEPW', + 'HOPPER', 'FRONTERA'] def __init__(self, host=None):