Update Ocean submit script

support/SubmitScripts/Ocean.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -
 #SBATCH -o spectre.stdout
-#SBATCH -e spectre.stderr
+#SBATCH -e spectre.stdout
 #SBATCH --ntasks-per-node 20
 #SBATCH -J KerrSchild
 #SBATCH --nodes 2
@@ -16,6 +16,11 @@
 #   number of nodes, and wall time limit in HH:MM:SS, respectively.
 # - Set the build directory, run directory, executable name,
 #   and input file below.
+# - Add a file ${HOME}/.charmrunrc that does the following (replace
+#   ${SPECTRE_BUILD_DIR}/../ with the path to your spectre checkout):
+#    source /etc/profile.d/lmod.sh
+#    source ${SPECTRE_BUILD_DIR}/../support/Environments/ocean_clang.sh
+#    spectre_load_modules
 #
 # NOTE: The executable will not be copied from the build directory, so if you
 #       update your build directory this file will use the updated executable.
@@ -26,24 +31,31 @@
 # To submit the script to the queue run:
 #   sbatch Ocean.sh
 
-# Replace these paths with the path to your build directory and to the
-# directory where you want the output to appear, i.e. the run directory
+# Replace these paths with the path to your build directory, to the source root
+# directory, the spectre dependencies module directory, and to the directory
+# where you want the output to appear, i.e. the run directory.
 # E.g., if you cloned spectre in your home directory, set
 # SPECTRE_BUILD_DIR to ${HOME}/spectre/build. If you want to run in a
 # directory called "Run" in the current directory, set
 # SPECTRE_RUN_DIR to ${PWD}/Run
-export SPECTRE_BUILD_DIR=${HOME}/Codes/spectre/spectre/build_singularity_release
+export SPECTRE_BUILD_DIR=${HOME}/spectre/build
+#export SPECTRE_MODULE_DIR=${HOME}/Codes/spectre_deps/modules/
 export SPECTRE_RUN_DIR=${PWD}/Run
 
 # Choose the executable and input file to run
 # To use an input file in the current directory, set
 # SPECTRE_INPUT_FILE to ${PWD}/InputFileName.yaml
-export SPECTRE_EXECUTABLE=${SPECTRE_BUILD_DIR}/bin/EvolveGeneralizedHarmonic
+export SPECTRE_EXECUTABLE=${SPECTRE_BUILD_DIR}/bin/EvolveGhSingleBlackHole
 export SPECTRE_INPUT_FILE=${PWD}/KerrSchild.yaml
 
 # These commands load the relevant modules and cd into the run directory,
 # creating it if it doesn't exist
 module load ohpc
+source ${SPECTRE_BUILD_DIR}/../support/Environments/ocean_gcc.sh
+module use ${SPECTRE_MODULE_DIR}
+spectre_load_modules
+module list
+
 mkdir -p ${SPECTRE_RUN_DIR}
 cd ${SPECTRE_RUN_DIR}
 
@@ -59,11 +71,67 @@ export PATH=${SPECTRE_BUILD_DIR}/bin:$PATH
 # Flag to stop blas in CCE from parallelizing without charm++
 export OPENBLAS_NUM_THREADS=1
 
-# The 19 is there because Charm++ uses one thread per node for communication
-# Here, -np should take the number of nodes (must be the same as --nodes
+# Generate the nodefile
+echo "Running on the following nodes:"
+echo ${SLURM_NODELIST}
+touch nodelist.$SLURM_JOBID
+for node in $(echo $SLURM_NODELIST | scontrol show hostnames); do
+  echo "host ${node}" >> nodelist.$SLURM_JOBID
+done
+
+# The (SLURM_NTASKS_PER_NODE - 1) is there because Charm++ uses one thread per
+# node for communication
+# Here, ++np should take the number of nodes (must be the same as --nodes
 # in the #SBATCH options above).
-SPECTRE_COMMAND="${SPECTRE_EXECUTABLE} +ppn 19 +pemap 0-18 +commap 19"
+WORKER_THREADS_PER_NODE=$((SLURM_NTASKS_PER_NODE - 1))
+WORKER_THREADS=$((SLURM_NPROCS - SLURM_NNODES))
+SPECTRE_COMMAND="${SPECTRE_EXECUTABLE} ++np ${SLURM_NNODES} \
+++p ${WORKER_THREADS} ++ppn ${WORKER_THREADS_PER_NODE} \
+++nodelist nodelist.${SLURM_JOBID}"
+
+
+# When invoking through `charmrun`, charm will initiate remote sessions which
+# will wipe out environment settings unless it is forced to re-initialize the
+# spectre environment between the start of the remote session and starting the
+# spectre executable
+echo "#!/bin/sh
+source ${SPECTRE_BUILD_DIR}/../support/Environments/ocean_gcc.sh
+module use ${SPECTRE_MODULE_DIR}
+spectre_load_modules
+\$@
+" > ${SPECTRE_RUN_DIR}/runscript.${SLURM_JOBID}
+
+chmod u+x ${SPECTRE_RUN_DIR}/runscript.${SLURM_JOBID}
 
-mpirun -np ${SLURM_JOB_NUM_NODES} --map-by ppr:1:node singularity exec \
-/opt/ohpc/pub/containers/spectre_ocean.sif \
-bash -c "${SPECTRE_COMMAND} --input-file ${SPECTRE_INPUT_FILE}"
+checkpoints=0
+current_checkpoint=0000
+if [[ $checkpoints == 0 ]]; then
+  charmrun ++runscript ${SPECTRE_RUN_DIR}/runscript.${SLURM_JOBID} \
+          ${SPECTRE_COMMAND} --input-file ${SPECTRE_INPUT_FILE}
+  sleep 10s
+  # If a checkpoint is found add one to checkpoint and sumbit next job
+  if test -e "${PWD}/Checkpoints/Checkpoint_$current_checkpoint"; then
+    cp ../Ocean.sh .
+    sed -i "s/^checkpoints=0/checkpoints=1/" Ocean.sh
+    sbatch Ocean.sh
+  fi
+# Section to start from checkpoint
+elif [[ $checkpoints -gt 0 && $checkpoints -lt 10000 ]]; then
+  ln -s ${PWD}/../Checkpoints/Checkpoint_$current_checkpoint .
+  charmrun ${SPECTRE_COMMAND} \
+          +restart Checkpoints/Checkpoint_$current_checkpoint \
+          --input-file ${SPECTRE_INPUT_FILE}
+  sleep 10s
+  # If the next checkpoint was created modify variables and submit next job
+  printf -v next_checkpoint %04d $checkpoints
+  if test -e "${PWD}/Checkpoints/Checkpoint_$next_checkpoint"; then
+    cp ../Ocean.sh .
+    next_num_of_checkpoints=$(($checkpoints + 1))
+    #Updating variables for the next possible checkpoint
+    sed -i "s/^checkpoints=$checkpoints/"\
+"checkpoints=$next_num_of_checkpoints/" Ocean.sh
+    sed -i "s/^current_checkpoint=$current_checkpoint/"\
+"current_checkpoint=$next_checkpoint/" Ocean.sh
+    sbatch Ocean.sh
+  fi
+fi