diff --git a/README.betzy b/README.betzy index 898980f3..fbd3eba3 100644 --- a/README.betzy +++ b/README.betzy @@ -1,16 +1,18 @@ !+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -1) Recommended .bashrc content for BETZY +1) Recommended environment for BETZY -ml purge -ml load ESMF/8.3.0-iomkl-2022a -ml load UDUNITS/2.2.28-GCCcore-11.3.0 -ml load Python/3.10.4-GCCcore-11.3.0 -ml load GSL/2.7-intel-compilers-2022.1.0 -ml load FFTW/3.3.10-GCC-11.3.0 -ml load CMake/3.23.1-GCCcore-11.3.0 + ml purge + ml load ESMF/8.3.0-iomkl-2022a + ml load UDUNITS/2.2.28-GCCcore-11.3.0 + ml load Python/3.10.4-GCCcore-11.3.0 + ml load GSL/2.7-intel-compilers-2022.1.0 + ml load FFTW/3.3.10-GCC-11.3.0 + ml load CMake/3.23.1-GCCcore-11.3.0 -ulimit -s 2000000 ## if needed + ulimit -s 2000000 ## if needed + execute the following line to get the environment (adjust the path if needed): + source ~/NERSC-HYCOM-CICE/betzy_env.sh 2) Compiling hycom_all under NERSC-HYCOM-CICE/hycom/hycom_ALL/hycom_2.2.72_ALL diff --git a/README.fram b/README.fram index e247d781..aff5b2bc 100644 --- a/README.fram +++ b/README.fram @@ -1,14 +1,16 @@ !+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -1) Recommended .bashrc content for FRAM - -ml purge -module load Python/3.10.4-GCCcore-11.3.0 -module load ESMF/8.3.0-intel-2022a -module load FFTW/3.3.10-GCC-11.3.0 -module load UDUNITS/2.2.28-GCCcore-11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load intel/2022a - +1) Recommended environment for FRAM + + ml purge + module load Python/3.10.4-GCCcore-11.3.0 + module load ESMF/8.3.0-intel-2022a + module load FFTW/3.3.10-GCC-11.3.0 + module load UDUNITS/2.2.28-GCCcore-11.3.0 + module load CMake/3.23.1-GCCcore-11.3.0 + module load intel/2022a + + execute the following line to get the environment (adjust the path if needed): + source ~/NERSC_HYCOM_CICE/fram_env.sh 2) Compiling hycom_all under NERSC-HYCOM-CICE/hycom/hycom_ALL/hycom_2.2.72_ALL diff --git a/TP2a0.10/expt_01.0/srjob.sh b/TP2a0.10/expt_01.0/srjob.sh index 2e43106a..d5f4a461 100755 --- a/TP2a0.10/expt_01.0/srjob.sh +++ b/TP2a0.10/expt_01.0/srjob.sh @@ -1,101 +1,64 @@ #!/bin/bash -l -#SBATCH --account=nn9481k - -#SBATCH -J TP2a010 - -#SBATCH -N 10 # number of nodes -#SBATCH -n 22 # number of nodes -## Each compute node has 24 cores (See more details in section Hardware on Sisu User Guide). - -##SBATCH -p small_long -## Choose a suitable queue -## How to check queue limits: scontrol show part -## for example: scontrol show part small - -## System message output file -#SBATCH -o log/HYCICE.%J.out - -## System error message file -#SBATCH -e log/HYCICE.%J.err - -## How long job takes, wallclock time hh:mm:ss -#SBATCH -t 00:30:00 - -#SBATCH --mail-type=END -#SBATCH --mail-user=jiping.xie@nersc.no - -## option: -n (total number of mpi processes) -## option: -N (number of mpi processes per compute node) -## option: -S (number of mpi processes per NUMA node) -## option: -ss (allocate memory only from a local NUMA node) - -## Calculate the total number of cores and store it in variable ncores -##(( ncores = SLURM_NNODES * 24 )) -##(( ncores = SLURM_NNODES * 32 )) #on Fram - -module load NCL/6.6.2-intel-2018b -module load FFTW/3.3.8-intel-2018b -module load Python/2.7.15-intel-2018b - - -export NMPI=220 +#SBATCH --account=nn9481k +#SBATCH --job-name=TP2a010 +#SBATCH --time="00:06:00" +#SBATCH --nodes=4 # number of nodes +#SBATCH --ntasks=504 # number of cores +#SBATCH --mail-type=END + +#SBATCH -o log/HY_CICE.out +#SBATCH -e log/HY_CICE.err + +# +export NMPI=504 export SLURM_SUBMIT_DIR=$(pwd) # Enter directory from where the job was submitted -cd $SLURM_SUBMIT_DIR || { echo "Could not go to dir $SLURM_SUBMIT_DIR "; exit 1; } +cd $SLURM_O_WORKDIR || { echo "Could not go to dir $SLURM_O_WORKDIR "; exit 1; } # ------------------- Fetch Environment ------------------------------ # -------- these are needed in preprocess scripts--------------------- echo "SLURM_JOBID = $SLURM_JOBID " -echo "SLURM_SUBMIT_DIR= $SLURM_SUBMIT_DIR " +echo "SLURM_JOBNAME = $SLURM_JOBNAME " +echo "SLURM_SUBMIT_DIR= $SLURM_SUBMIT_KDIR " +echo "SLURM_TASKNUM = $SLURM_TASKNUM " +echo "SLURM_NUM_PPN = $SLURM_NUM_PPN " [ -z "$NOMP" ] && NOMP=0 # Enter directory from where the job was submitted cd $SLURM_SUBMIT_DIR || { echo "Could not go to dir $SLURM_O_WORKDIR "; exit 1; } - # Initialize environment (sets Scratch dir ($S), Data dir $D ++ ) source ../REGION.src || { echo "Could not source ../REGION.src "; exit 1; } source ./EXPT.src || { echo "Could not source EXPT.src"; exit 1; } -echo "NMPI =$NMPI (Number of MPI tasks needed for running job) " - - -for ik in `seq 1 1` ; do - if [ $ik -eq 0 ] ; then - START="2001-12-25T00:00:00" - END="2002-01-01T00:00:00" - INITFLG="--init" - cp -f ice_in.0 ice_in - else - INITFLG="" - cp -f ice_in.1 ice_in - if [ $ik -eq 1 ] ; then - START="2001-12-25T00:00:00" - END="2002-01-02T00:00:00" - else - START="2010-01-02T00:00:00" - END="2016-01-02T00:00:00" - fi - fi - echo "Start time in pbsjob.sh: $START" - echo "End time in pbsjob.sh: $END" +source $NHCROOT/environment/betzy_env.sh || { echo "Could not source betzy_env.sh "; exit 1; } +#source $NHCROOT/environment/fram_env.sh || { echo "Could not source fram_env.sh "; exit 1; } - # Generate atmospheric forcing : - atmo_synoptic.sh erai $START $END - - # Transfer data files to scratch - must be in "expt_XXX" dir for this script - expt_preprocess.sh $START $END $INITFLG || { echo "Preprocess had fatal errors "; exit 1; } - - # Enter Scratch/run dir and Run model - cd $S || { echo "Could not go to dir $S "; exit 1; } - ##aprun -n $NMPI -m 500M ./hycom_cice > ../log/hycom.%J.out 2>&1 - srun --mpi=pmi2 -n $NMPI --cpu_bind=cores ./hycom_cice - - # Cleanup and move data files to data directory - must be in "expt_XXX" dir for this script - cd $P || { echo "Could not go to dir $P "; exit 1; } - expt_postprocess.sh +echo "NMPI =$NMPI (Number of MPI tasks needed for running job) " -done +START="1990-01-01T00:00:00" +END="1990-01-05T00:00:00" +INITFLG="--init" +#INITFLG="" +echo "Start time in pbsjob.sh: $START" +echo "End time in pbsjob.sh: $END" +# Generate atmospheric forcing : +#atmo_synoptic.sh erai+all $START $END +../bin/atmo_synoptic.sh era5+lw $START $END + +# Transfer data files to scratch - must be in "expt_XXX" dir for this script +# expt_preprocess.sh $START $END $INITFLG || { echo "Preprocess had fatal errors "; exit 2; } +../bin/expt_preprocess.sh $START $END $INITFLG || { echo "Preprocess had fatal errors "; exit 1; } + +# Enter Scratch/run dir and Run model +cd $S || { echo "Could not go to dir $S "; exit 1; } +#srun -n $NMPI ./hycom_cice > ../log/hycom.${SLURM_JOBID}.out 2>&1 +srun -n $NMPI --cpu_bind=cores ./hycom_cice + +# Cleanup and move data files to data directory - must be in "expt_XXX" dir for this script +cd $P || { echo "Could not go to dir $P "; exit 1; } +expt_postprocess.sh exit $? + diff --git a/TP5a0.06/expt_01.1/EXPT.src b/TP5a0.06/expt_01.1/EXPT.src index d5f94a96..1da97662 100755 --- a/TP5a0.06/expt_01.1/EXPT.src +++ b/TP5a0.06/expt_01.1/EXPT.src @@ -1,4 +1,4 @@ -#!/binn/bash +#!/bin/bash # # Sets environment for this experiment, also makes the scratch (S) and # data (D) directories if not already present @@ -20,7 +20,7 @@ mydir=$(cd $(dirname ${BASH_SOURCE}) && pwd) unset -v X E T V K P D S X="01.1" # X based on dir name (expt_01.1) E="011" # E is X without "." -T="08" # Topography version +T="05" # Topography version export V=2.2.98 # hycom version #export K=`grep "'kdm ' =" blk* | awk '{printf("%03d", $1)}'` # get kdm from blkdat export K=`grep "'kdm ' =" $mydir/blkdat.input | awk '{printf("%03d", $1)}'` # get kdm from blkdat @@ -33,6 +33,7 @@ export S=$P/SCRATCH # Scratch area export SIGVER=1 # Version of equation of state (this is 7-term sigma 2). Must not cause conflict with thflag in blkdat.input export NMPI=336 export MXBLCKS=2 # The maximal number of ice blocks relates to the MPI cores distribution, and could be recommened by the error message in a model run. +export COMPILE_BIOMODEL="no" # Turn ECOSMO on with "yes" # Consistency check. Ensures expt dir ends in expt_X #echo $tmp diff --git a/TP5a0.06/topo/cice_grid.nc b/TP5a0.06/topo/cice_grid.nc new file mode 100755 index 00000000..c2c9cdd2 Binary files /dev/null and b/TP5a0.06/topo/cice_grid.nc differ diff --git a/bin/create_ref_case.sh b/bin/create_ref_case.sh index 743e08c6..de2fd469 100644 --- a/bin/create_ref_case.sh +++ b/bin/create_ref_case.sh @@ -131,7 +131,7 @@ if [ ${iceclim} -eq 1 ]; then ml load matplotlib/3.5.2-intel-2022a ${BINDIR}/Grid_Bathy/cice_kmt.py regional.depth.a fi - prg=${BINDIR}ice_climatology/extract_clim_iceh_update.sh + prg=${BINDIR}ice_climatology/extract_clim_iceh.sh ${prg} ${EDIR} cd ${EDIR} [ -r ice_clim ] && rm -rf ice_clim diff --git a/bin/expt_preprocess.sh b/bin/expt_preprocess.sh index 3333cc73..18919af4 100755 --- a/bin/expt_preprocess.sh +++ b/bin/expt_preprocess.sh @@ -288,9 +288,10 @@ ${pget} $BASEDIR/topo/regional.grid.a regional.grid.a || tellerror "no grid file ${pget} $BASEDIR/topo/regional.grid.b regional.grid.b || tellerror "no grid file regional.grid.a" ${pget} $BASEDIR/topo/depth_${R}_${T}.a regional.depth.a || tellerror "no topo file depth_${R}_${T}.a" ${pget} $BASEDIR/topo/depth_${R}_${T}.b regional.depth.b || tellerror "no topo file depth_${R}_${T}.b" -${pget} $BASEDIR/topo/kmt_${R}_${T}.nc cice_kmt.nc || tellerror "no kmt file $BASEDIR/topo/kmt_${R}_${T}.nc " -${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc " - +if [ $ICEFLG != 0 ] ; then + ${pget} $BASEDIR/topo/kmt_${R}_${T}.nc cice_kmt.nc || tellerror "no kmt file $BASEDIR/topo/kmt_${R}_${T}.nc " + ${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc " +fi if [ "$SSTRLX" -eq 3 ] ; then [ -f $CLMDIR/seatmp.a ] || tellerror "File $CLMDIR/seatmp.a does not exist" diff --git a/bin/ice_climatology/extract_clim_iceh.sh b/bin/ice_climatology/extract_clim_iceh.sh index 72a8d518..cc2905cc 100755 --- a/bin/ice_climatology/extract_clim_iceh.sh +++ b/bin/ice_climatology/extract_clim_iceh.sh @@ -87,7 +87,7 @@ if [ -s ${Fini} -a -s ${Ftmpmask} ]; then ml load CDO/2.0.6-gompi-2022a cdo remapbil,${Ftmpmask} ${Fini} ${Fout} - ml load NCO/5.1.3-iimpi-2022a + ml load NCO/5.1.9-iomkl-2022a echo "Defaulting the values for different masks ... " ncks -v ${Vars} ${Fout} ${Ftmp} ncrename -h -O -v siconc,aice_raw -v sithick,hi_raw ${Ftmp} diff --git a/contributors.txt b/contributors.txt index 47baac6d..427c7b38 100644 --- a/contributors.txt +++ b/contributors.txt @@ -6,3 +6,4 @@ Roshin Raj Annette Samuelsen Achref Othmani Till Rasmussen +Shuang Gao diff --git a/environment/betzy_env.sh b/environment/betzy_env.sh new file mode 100644 index 00000000..1e58521a --- /dev/null +++ b/environment/betzy_env.sh @@ -0,0 +1,10 @@ +# Environment for compiling/running HYCOM-CICE on betzy +ml purge +ml load ESMF/8.3.0-iomkl-2022a +ml load UDUNITS/2.2.28-GCCcore-11.3.0 +ml load Python/3.10.4-GCCcore-11.3.0 +ml load GSL/2.7-intel-compilers-2022.1.0 +ml load FFTW/3.3.10-GCC-11.3.0 +ml load CMake/3.23.1-GCCcore-11.3.0 + +ulimit -s 2000000 diff --git a/environment/fram_env.sh b/environment/fram_env.sh new file mode 100644 index 00000000..9a75ccd4 --- /dev/null +++ b/environment/fram_env.sh @@ -0,0 +1,8 @@ +# Environment for compiling/running HYCOM-CICE on fram +ml purge +module load Python/3.10.4-GCCcore-11.3.0 +module load ESMF/8.3.0-intel-2022a +module load FFTW/3.3.10-GCC-11.3.0 +module load UDUNITS/2.2.28-GCCcore-11.3.0 +module load CMake/3.23.1-GCCcore-11.3.0 +module load intel/2022a