Add testing for graceful shutdown more frequently

.github/workflows/temp-branch-build-and-push.yaml

@@ -3,7 +3,7 @@ name: Branch - Build and push docker image
 on:
   push:
     branches:
-      - "reduce-size-docker-image"
+      - "POP-2045/investigate-nodes-not-syncing-up"
 
 concurrency:
   group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'

iris-mpc/src/bin/server.rs

@@ -901,6 +901,15 @@ async fn server_main(config: Config) -> eyre::Result<()> {
                         "Node {} did not respond with success, starting graceful shutdown",
                         host
                     );
+                    // if the nodes are still starting up and they get a failure - we can panic and
+                    // not start graceful shutdown
+                    if last_response[i] == String::default() {
+                        panic!(
+                            "Node {} did not respond with success during heartbeat init phase, \
+                             killing server...",
+                            host
+                        );
+                    }
 
                     if !heartbeat_shutdown_handler.is_shutting_down() {
                         heartbeat_shutdown_handler.trigger_manual_shutdown();
@@ -968,6 +977,8 @@ async fn server_main(config: Config) -> eyre::Result<()> {
     tracing::info!("Heartbeat starting...");
     heartbeat_rx.await?;
     tracing::info!("Heartbeat on all nodes started.");
+    let download_shutdown_handler = Arc::clone(&shutdown_handler);
+
     background_tasks.check_tasks();
 
     let my_state = SyncState {
@@ -988,7 +999,6 @@ async fn server_main(config: Config) -> eyre::Result<()> {
     let load_chunks_parallelism = config.load_chunks_parallelism;
     let db_chunks_bucket_name = config.db_chunks_bucket_name.clone();
     let db_chunks_folder_name = config.db_chunks_folder_name.clone();
-    let download_shutdown_handler = Arc::clone(&shutdown_handler);
 
     let (tx, rx) = oneshot::channel();
     background_tasks.spawn_blocking(move || {
@@ -1033,6 +1043,11 @@ async fn server_main(config: Config) -> eyre::Result<()> {
             metrics::counter!("db.sync.rollback").increment(1);
         }
 
+        if download_shutdown_handler.is_shutting_down() {
+            tracing::warn!("Shutting down has been triggered");
+            return Ok(());
+        }
+
         // --------------------------------------------------------------------------
         // ANCHOR: Load the database
         // --------------------------------------------------------------------------