Merge pull request sxs-collaboration#6463 from nilsvu/fix_status

Status CLI / dashboard: handle pending jobs better, plot simulation time over calendar time

src/Parallel/PhaseControl/CheckpointAndExitAfterWallclock.hpp

@@ -24,6 +24,7 @@
 #include "Utilities/System/ParallelInfo.hpp"
 #include "Utilities/TMPL.hpp"
 #include "Utilities/TaggedTuple.hpp"
+#include "Utilities/UtcTime.hpp"
 
 namespace PhaseControl {
 
@@ -235,6 +236,8 @@ CheckpointAndExitAfterWallclock::arbitrate_phase_change_impl(
     } else {
       // if current_phase is WriteCheckpoint, we follow with updating options
       if (current_phase == Parallel::Phase::WriteCheckpoint) {
+        Parallel::printf("Restarting from checkpoint. Date and time: %s\n",
+                         utc_time());
         return std::make_pair(
             Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint,
             ArbitrationStrategy::PermitAdditionalJumps);

src/Utilities/CMakeLists.txt

@@ -19,6 +19,7 @@ spectre_target_sources(
   OptimizerHacks.cpp
   PrettyType.cpp
   Rational.cpp
+  UtcTime.cpp
   WrapText.cpp
   )
 
@@ -83,6 +84,7 @@ spectre_target_headers(
   Tuple.hpp
   TupleSlice.hpp
   TypeTraits.hpp
+  UtcTime.hpp
   VectorAlgebra.hpp
   WrapText.hpp
   )

src/Utilities/UtcTime.cpp

@@ -0,0 +1,21 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#include "Utilities/UtcTime.hpp"
+
+#include <sstream>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/date_time/posix_time/posix_time_io.hpp>
+
+std::string utc_time() {
+  static const std::string datetime_format = "%Y-%m-%d %H:%M:%S UTC";
+  // Use this needlessly complicated implementation with Boost because the
+  // needed features from C++20 aren't available yet in all compilers.
+  const auto now_utc = boost::posix_time::second_clock::universal_time();
+  auto* time_facet = new boost::posix_time::time_facet(datetime_format.c_str());
+  std::ostringstream oss;
+  oss.imbue(std::locale(std::locale::classic(), time_facet));
+  oss << now_utc;
+  return oss.str();
+}

src/Utilities/UtcTime.hpp

@@ -0,0 +1,13 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <string>
+
+/*!
+ * \brief Get the current date and time in UTC.
+ *
+ * The date and time are formatted as "YYYY-MM-DD HH:MM:SS UTC".
+ */
+std::string utc_time();

tools/Status/ExecutableStatus/EvolveGhBinaryBlackHole.py

@@ -116,6 +116,9 @@ def render_dashboard(self, job: dict, input_file: dict, metadata: dict):
 
         run_dir = Path(job["WorkDir"])
         reduction_files = list_reduction_files(job=job, input_file=input_file)
+        if not reduction_files:
+            st.warning("No data yet.")
+            return
 
         # Common horizon
         with h5py.File(run_dir / reduction_files[-1], "r") as open_h5file:

tools/Status/ExecutableStatus/ExecutableStatus.py

@@ -1,6 +1,7 @@
 # Distributed under the MIT License.
 # See LICENSE.txt for details.
 
+import datetime
 import logging
 import os
 from pathlib import Path
@@ -21,12 +22,36 @@
 def list_reduction_files(job: dict, input_file: dict):
     reductions_file_name = input_file["Observers"]["ReductionFileName"] + ".h5"
     if job["SegmentsDir"]:
-        return [
+        reduction_files = [
             segment_dir.path / reductions_file_name
             for segment_dir in list_segments(job["SegmentsDir"])
         ]
     else:
-        return [Path(job["WorkDir"]) / reductions_file_name]
+        reduction_files = [Path(job["WorkDir"]) / reductions_file_name]
+    return list(
+        filter(
+            lambda reduction_file: reduction_file.exists(),
+            reduction_files,
+        )
+    )
+
+
+def get_start_time(spectre_out: Path):
+    with open(spectre_out, "r") as f:
+        for line in f:
+            if (
+                "Date and time at startup:" in line
+                or "Restarting from checkpoint. Date and time:" in line
+            ):
+                dt = pd.to_datetime(line.split(":", 1)[-1].strip())
+                if dt.tzinfo:
+                    # Convert to local time zone
+                    dt = dt.tz_convert(
+                        datetime.datetime.now().astimezone().tzinfo
+                    )
+                return dt
+
+    return None
 
 
 class ExecutableStatus:
@@ -204,10 +229,14 @@ def format(self, field, value):
             return f"{value:g}"
         raise ValueError
 
-    def render_time_steps(self, input_file: dict, reduction_files: list):
+    def render_time_steps(self, input_file: dict, reduction_files: List[Path]):
         import plotly.express as px
         import streamlit as st
 
+        if not reduction_files:
+            st.warning("No data yet.")
+            return
+
         try:
             observe_time_event = find_event(
                 "ObserveTimeStep", "EventsAndTriggersAtSlabs", input_file
@@ -238,7 +267,8 @@ def get_time_steps(reductions_file):
         )
 
         st.subheader("Time steps")
-        time_steps = pd.concat(map(get_time_steps, reduction_files))
+        all_time_steps = list(map(get_time_steps, reduction_files))
+        time_steps = pd.concat(all_time_steps)
         fig = px.line(
             time_steps,
             y=[
@@ -258,7 +288,10 @@ def get_time_steps(reductions_file):
         run_speed = (
             time_steps.index.diff() / time_steps["Maximum Walltime"].diff()
         ) * 3600
-        fig = px.line(run_speed.rolling(30, min_periods=1).mean())
+        avg_run_speed = run_speed.rolling(30, min_periods=1).mean()
+
+        # Plot simulation speed
+        fig = px.line(avg_run_speed)
         fig.add_scatter(
             x=run_speed.index,
             y=run_speed,
@@ -273,6 +306,33 @@ def get_time_steps(reductions_file):
         )
         st.plotly_chart(fig)
 
+        # Plot simulation speed over calendar time
+        import plotly.graph_objects as go
+
+        fig = go.Figure()
+        for reduction_file, time_steps in zip(reduction_files, all_time_steps):
+            spectre_out = reduction_file.parent / "spectre.out"
+            start_time = get_start_time(spectre_out)
+            if not start_time:
+                continue
+            calendar_time = start_time + pd.to_timedelta(
+                time_steps["Maximum Walltime"], unit="s"
+            )
+            fig.add_trace(
+                go.Scatter(
+                    x=calendar_time,
+                    y=time_steps.index,
+                    mode="lines",
+                    name=reduction_file.parent.name,
+                )
+            )
+        fig.update_layout(
+            xaxis_title="Calendar day",
+            yaxis_title="Simulation time [M]",
+            showlegend=False,
+        )
+        st.plotly_chart(fig)
+
     def render_dashboard(self, job: dict, input_file: dict, metadata: dict):
         return self.render_time_steps(
             input_file, list_reduction_files(job, input_file)

tools/Status/Status.py

@@ -65,7 +65,10 @@ def fetch_job_data(
     except subprocess.CalledProcessError as err:
         raise ValueError(completed_process.stderr) from err
     job_data = pd.read_table(
-        StringIO(completed_process.stdout), sep="|", keep_default_na=False
+        StringIO(completed_process.stdout),
+        sep="|",
+        keep_default_na=False,
+        on_bad_lines="warn",
     )
     # Parse dates and times. Do this in postprocessing because
     # `pd.read_table(parse_dates=...)` doesn't handle NaN values well.