diff --git a/support/Environments/urania.sh b/support/Environments/urania.sh new file mode 100644 index 000000000000..1544ca40ee21 --- /dev/null +++ b/support/Environments/urania.sh @@ -0,0 +1,69 @@ +#!/bin/env sh + +# Distributed under the MIT License. +# See LICENSE.txt for details. + +spectre_setup_charm_paths() { + # Define Charm paths + export CHARM_ROOT=/u/guilara/charm_impi_2/mpi-linux-x86_64-smp + export PATH=$PATH:/u/guilara/charm_impi_2/mpi-linux-x86_64-smp/bin +} + +spectre_load_modules() { + module load gcc/11 + module load impi/2021.7 + module load boost/1.79 + module load gsl/1.16 + module load cmake/3.26 + module load hdf5-serial/1.12.2 + module load anaconda/3/2021.11 + module load paraview/5.10 + # Load Spack environment + source /u/guilara/repos/spack/share/spack/setup-env.sh + source /urania/u/guilara/repos/spack/var/spack/environments\ +/env3_spectre_impi/loads + # Load python environment + source /u/guilara/envs/spectre_env +} + +spectre_unload_modules() { + module unload gcc/11 + module unload impi/2021.7 + module unload boost/1.79 + module unload gsl/1.16 + module unload cmake/3.26 + module unload hdf5-serial/1.12.2 + module unload anaconda/3/2021.11 + module unload paraview/5.10 + # Unload Spack environment + spack env deactivate + # Unload python environment + deactivate +} + +spectre_run_cmake() { + if [ -z ${SPECTRE_HOME} ]; then + echo "You must set SPECTRE_HOME to the cloned SpECTRE directory" + return 1 + fi + spectre_load_modules + + cmake -D CMAKE_C_COMPILER=gcc \ + -D CMAKE_CXX_COMPILER=g++ \ + -D CMAKE_Fortran_COMPILER=gfortran \ + -D CHARM_ROOT=$CHARM_ROOT \ + -D CMAKE_BUILD_TYPE=Release \ + -D DEBUG_SYMBOLS=OFF \ + -D BUILD_SHARED_LIBS=ON \ + -D MEMORY_ALLOCATOR=JEMALLOC \ + -D BUILD_PYTHON_BINDINGS=ON \ + -D MACHINE=Urania \ + -D Catch2_DIR=/u/guilara/repos/Catch2/install_dir/lib64/cmake/Catch2 + -D MPI_C_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpigcc \ + -D MPI_CXX_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpig++ \ + -D MPI_Fortran_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpigfortran \ + -Wno-dev "$@" $SPECTRE_HOME +} diff --git a/support/Machines/Urania.yaml b/support/Machines/Urania.yaml new file mode 100644 index 000000000000..786d27fcf930 --- /dev/null +++ b/support/Machines/Urania.yaml @@ -0,0 +1,11 @@ +# Distributed under the MIT License. +# See LICENSE.txt for details. + +Machine: + Name: Urania + Description: | + Supercomputer at the Max Planck Computing Data Facilty. + DefaultProcsPerNode: 72 + DefaultQueue: "p.urania" + DefaultTimeLimit: "1-00:00:00" + LaunchCommandSingleNode: ["srun", "-n", "1"] diff --git a/support/Python/Schedule.py b/support/Python/Schedule.py index ab3b555420d7..e5be7775e65f 100644 --- a/support/Python/Schedule.py +++ b/support/Python/Schedule.py @@ -149,6 +149,7 @@ def schedule( submit: Optional[bool] = None, clean_output: bool = False, force: bool = False, + validate: Optional[bool] = True, extra_params: dict = {}, **kwargs, ) -> Optional[subprocess.CompletedProcess]: @@ -319,6 +320,8 @@ def schedule( files in the 'run_dir' before scheduling the run. (Default: 'False') force: Optional. When 'True', overwrite input file and submit script in the 'run_dir' instead of raising an error when they already exist. + validate: Optional. When 'True', validate that the input file is parsed + correctly. When 'False' skip this step. extra_params: Optional. Dictionary of extra parameters passed to input file and submit script templates. Parameters can also be passed as keyword arguments to this function instead. @@ -549,9 +552,11 @@ def schedule( ) # Validate input file - validate_input_file( - input_file_path.resolve(), executable=executable, work_dir=run_dir - ) + if validate: + validate_input_file( + input_file_path.resolve(), executable=executable, work_dir=run_dir + ) + # - If the input file may request resubmissions, make sure we have a # segments directory metadata, input_file = yaml.safe_load_all(rendered_input_file) @@ -861,6 +866,11 @@ def scheduler_options(f): "You may also want to use '--clean-output'." ), ) + @click.option( + "--validate/--no-validate", + default=True, + help="Validate or skip the validation of the input file.", + ) # Scheduling options @click.option( "--scheduler", diff --git a/support/SubmitScripts/Urania.sh b/support/SubmitScripts/Urania.sh new file mode 100644 index 000000000000..c6ee572518b2 --- /dev/null +++ b/support/SubmitScripts/Urania.sh @@ -0,0 +1,39 @@ +{% extends "SubmitTemplateBase.sh" %} + +# Distributed under the MIT License. +# See LICENSE.txt for details. + +# Uranina -- HPC cluster of ACR division of MPI for Grav Physics, housed at the +# Max Planck Computing & Data Facility. +# https://docs.mpcdf.mpg.de/doc/computing/clusters/systems/Gravitational_Physics_ACR.html + +{% block head %} +{{ super() -}} +#SBATCH --nodes {{ num_nodes | default(1) }} +#SBATCH --ntasks-per-node=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --cpus-per-task=72 +#SBATCH -t {{ time_limit | default("1-00:00:00") }} +#SBATCH -p {{ queue | default("p.urania") }} +{% endblock %} + +{% block charm_ppn %} +# Two thread for communication +CHARM_PPN=$(expr ${SLURM_CPUS_PER_TASK} - 2) +{% endblock %} + +{% block list_modules %} +# Load compiler and MPI modules with explicit version specifications, +# consistently with the versions used to build the executable. +source ${SPECTRE_HOME}/support/Environments/urania.sh +spectre_load_modules +spectre_setup_charm_paths + +{% endblock %} + +{% block run_command %} +srun -n ${SLURM_NTASKS} ${SPECTRE_EXECUTABLE} \ + --input-file ${SPECTRE_INPUT_FILE} \ + ++ppn ${CHARM_PPN} +pemap 0-34,36-70 +commap 35,71 \ + ${SPECTRE_CHECKPOINT:+ +restart "${SPECTRE_CHECKPOINT}"} +{% endblock %} diff --git a/tests/support/Python/Test_Schedule.py b/tests/support/Python/Test_Schedule.py index 4db7c57c7c32..42b50942af1f 100644 --- a/tests/support/Python/Test_Schedule.py +++ b/tests/support/Python/Test_Schedule.py @@ -230,6 +230,7 @@ def test_schedule(self): extra_option="TestOpt", metadata_option="MetaOpt", force=False, + validate=True, input_file="InputFile.yaml", input_file_name="InputFile.yaml", input_file_template=str(self.test_dir / "InputFile.yaml"),