diff --git a/.github/workflows/temp-branch-build-and-push.yaml b/.github/workflows/temp-branch-build-and-push.yaml index fc150d0b2..77dd37eff 100644 --- a/.github/workflows/temp-branch-build-and-push.yaml +++ b/.github/workflows/temp-branch-build-and-push.yaml @@ -3,7 +3,7 @@ name: Branch - Build and push docker image on: push: branches: - - "add-graceful-shutdown-during-s3-sync" + - "POP-2045/investigate-nodes-not-syncing-up" concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' diff --git a/iris-mpc/src/bin/server.rs b/iris-mpc/src/bin/server.rs index 6b24058e9..566e1c2ba 100644 --- a/iris-mpc/src/bin/server.rs +++ b/iris-mpc/src/bin/server.rs @@ -969,6 +969,12 @@ async fn server_main(config: Config) -> eyre::Result<()> { tracing::info!("Heartbeat starting..."); heartbeat_rx.await?; tracing::info!("Heartbeat on all nodes started."); + let download_shutdown_handler = Arc::clone(&shutdown_handler); + + if download_shutdown_handler.is_shutting_down() { + tracing::warn!("Shutting down has been triggered"); + return Ok(()); + } background_tasks.check_tasks(); let my_state = SyncState { @@ -989,7 +995,6 @@ async fn server_main(config: Config) -> eyre::Result<()> { let load_chunks_parallelism = config.load_chunks_parallelism; let db_chunks_bucket_name = config.db_chunks_bucket_name.clone(); let db_chunks_folder_name = config.db_chunks_folder_name.clone(); - let download_shutdown_handler = Arc::clone(&shutdown_handler); let (tx, rx) = oneshot::channel(); background_tasks.spawn_blocking(move || { @@ -1034,6 +1039,11 @@ async fn server_main(config: Config) -> eyre::Result<()> { metrics::counter!("db.sync.rollback").increment(1); } + if download_shutdown_handler.is_shutting_down() { + tracing::warn!("Shutting down has been triggered"); + return Ok(()); + } + // -------------------------------------------------------------------------- // ANCHOR: Load the database // --------------------------------------------------------------------------