Add trigger for analysis restarts

parthenon-hpc-lab · Oct 16, 2023 · 0931c72 · 0931c72
1 parent 4a4d6ae
commit 0931c72
Show file tree

Hide file tree

Showing 8 changed files with 35 additions and 12 deletions.
diff --git a/src/argument_parser.hpp b/src/argument_parser.hpp
@@ -54,6 +54,12 @@ class ArgParse {
           res_flag = 1;
           restart_filename = argv[++i];
           break;
+        case 'a': // -a <restart_file>
+          invalid = invalid_arg();
+          res_flag = 1;
+          analysis_flag = 1;
+          restart_filename = argv[++i];
+          break;
         case 'd': // -d <run_directory>
           invalid = invalid_arg();
           prundir = argv[++i];
@@ -81,6 +87,7 @@ class ArgParse {
             std::cout << "Options:" << std::endl;
             std::cout << "  -i <file>       specify input file [athinput]\n";
             std::cout << "  -r <file>       restart with this file\n";
+            std::cout << "  -a <file>       analyze/postprocess this file\n";
             std::cout << "  -d <directory>  specify run dir [current dir]\n";
             std::cout << "  -n              parse input file and quit\n";
             std::cout << "  -c              show configuration and quit\n";
@@ -119,6 +126,7 @@ class ArgParse {
   char *input_filename = nullptr;
   char *restart_filename = nullptr;
   char *prundir = nullptr;
+  int analysis_flag = 0;
   int res_flag = 0;
   int narg_flag = 0;
   int iarg_flag = 0;

diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp
@@ -65,7 +65,8 @@ DriverStatus EvolutionDriver::Execute() {
   PreExecute();
   InitializeBlockTimeStepsAndBoundaries();
   SetGlobalTimeStep();
-  OutputSignal signal = OutputSignal::none;
+  OutputSignal signal =
+      pmesh->analysis_flag ? OutputSignal::analysis : OutputSignal::none;
   pouts->MakeOutputs(pmesh, pinput, &tm, signal);
   pmesh->mbcnt = 0;
   int perf_cycle_offset =
@@ -76,7 +77,7 @@ DriverStatus EvolutionDriver::Execute() {
   DumpInputParameters();
 
   Kokkos::Profiling::pushRegion("Driver_Main");
-  while (tm.KeepGoing()) {
+  while (tm.KeepGoing() && signal != OutputSignal::analysis) {
     if (Globals::my_rank == 0) OutputCycleDiagnostics();
 
     pmesh->PreStepUserWorkInLoop(pmesh, pinput, tm);

diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp
@@ -94,13 +94,14 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
                   pin->GetOrAddString("parthenon/mesh", "refinement", "none") == "static")
                      ? true
                      : false),
-      nbnew(), nbdel(), step_since_lb(), gflag(), packages(packages),
+      nbnew(), nbdel(), analysis_flag(false), step_since_lb(), gflag(),
+      packages(packages),
       // private members:
       num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)),
       tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true),
-      lb_automatic_(), lb_manual_(), MeshGenerator_{nullptr, UniformMeshGenerator<X1DIR>,
-                                                    UniformMeshGenerator<X2DIR>,
-                                                    UniformMeshGenerator<X3DIR>},
+      lb_automatic_(), lb_manual_(),
+      MeshGenerator_{nullptr, UniformMeshGenerator<X1DIR>, UniformMeshGenerator<X2DIR>,
+                     UniformMeshGenerator<X3DIR>},
       MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} {
   std::stringstream msg;
   RegionSize block_size;
@@ -515,13 +516,14 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr,
                   pin->GetOrAddString("parthenon/mesh", "refinement", "none") == "static")
                      ? true
                      : false),
-      nbnew(), nbdel(), step_since_lb(), gflag(), packages(packages),
+      nbnew(), nbdel(), analysis_flag(false), step_since_lb(), gflag(),
+      packages(packages),
       // private members:
       num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)),
       tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true),
-      lb_automatic_(), lb_manual_(), MeshGenerator_{nullptr, UniformMeshGenerator<X1DIR>,
-                                                    UniformMeshGenerator<X2DIR>,
-                                                    UniformMeshGenerator<X3DIR>},
+      lb_automatic_(), lb_manual_(),
+      MeshGenerator_{nullptr, UniformMeshGenerator<X1DIR>, UniformMeshGenerator<X2DIR>,
+                     UniformMeshGenerator<X3DIR>},
       MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} {
   std::stringstream msg;
   RegionSize block_size;

diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp
@@ -100,6 +100,7 @@ class Mesh {
   const bool adaptive, multilevel;
   int nbtotal, nbnew, nbdel;
   std::uint64_t mbcnt;
+  bool analysis_flag; // flag if this mesh is constructed for postprocessing
 
   int step_since_lb;
   int gflag;

diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp
@@ -76,6 +76,7 @@
 #include "parameter_input.hpp"
 #include "parthenon_arrays.hpp"
 #include "utils/error_checking.hpp"
+#include "utils/utils.hpp"
 
 namespace parthenon {
 
@@ -146,6 +147,8 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) {
       // read cartesian mapping option
       op.cartesian_vector = false;
 
+      op.analysis_flag = pin->GetOrAddBoolean(op.block_name, "analysis_output", false);
+
       // read single precision output option
       const bool is_hdf5_output = (op.file_type == "rst") || (op.file_type == "hdf5");
 
@@ -420,7 +423,10 @@ void Outputs::MakeOutputs(Mesh *pm, ParameterInput *pin, SimTime *tm,
     if ((tm == nullptr) ||
         ((ptype->output_params.dt >= 0.0) &&
          ((tm->ncycle == 0) || (tm->time >= ptype->output_params.next_time) ||
-          (tm->time >= tm->tlim) || (signal != SignalHandler::OutputSignal::none)))) {
+          (tm->time >= tm->tlim) || (signal == SignalHandler::OutputSignal::now) ||
+          (signal == SignalHandler::OutputSignal::final) ||
+          (signal == SignalHandler::OutputSignal::analysis &&
+           ptype->output_params.analysis_flag)))) {
       if (first && ptype->output_params.file_type != "hst") {
         pm->ApplyUserWorkBeforeOutput(pin);
         first = false;

diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp
@@ -50,6 +50,7 @@ struct OutputParameters {
   std::string file_basename;
   int file_number_width;
   bool file_label_final;
+  bool analysis_flag; // write this output for analysis/postprocessing restarts
   std::string file_id;
   std::vector<std::string> variables;
   std::vector<std::string> component_labels;

diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp
@@ -182,6 +182,10 @@ void ParthenonManager::ParthenonInitPackagesAndMesh() {
     // close hdf5 file to prevent HDF5 hangs and corrupted files
     // if code dies after restart
     restartReader = nullptr;
+
+    if (arg.analysis_flag == 1) {
+      pmesh->analysis_flag = true;
+    }
   }
 
   // add root_level to all max_level

diff --git a/src/utils/utils.hpp b/src/utils/utils.hpp
@@ -70,7 +70,7 @@ std::string StringPrintf(const std::string &fmt, Args... args) {
 //  \brief static data and functions that implement a simple signal handling system
 namespace SignalHandler {
 
-enum class OutputSignal { none, now, final };
+enum class OutputSignal { none, now, final, analysis };
 constexpr int nsignal = 3;
 // using the +1 for signaling based on "output_now" trigger
 static volatile int signalflag[nsignal + 1];