Ensure heartbeat has enough time to start (#469)

worldcoin · Sep 27, 2024 · 1c0567f · 1c0567f
1 parent 3b1a7e4
commit 1c0567f
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 3 deletions.
diff --git a/deploy/stage/common-values-iris-mpc.yaml b/deploy/stage/common-values-iris-mpc.yaml
@@ -1,4 +1,4 @@
-image: "ghcr.io/worldcoin/iris-mpc:v0.8.15"
+image: "ghcr.io/worldcoin/iris-mpc:v0.8.16"
 
 environment: stage
 replicaCount: 1

diff --git a/iris-mpc-gpu/src/server/heartbeat_nccl.rs b/iris-mpc-gpu/src/server/heartbeat_nccl.rs
@@ -66,7 +66,7 @@ pub async fn start_heartbeat(
         }
     });
 
-    let mut timeout_interval = 2
+    let mut timeout_interval = 10
         * NCCL_START_WAIT_TIME
         * (NCCL_START_RETRIES - 1).try_into()?
         * DeviceManager::init().device_count().try_into()?;

diff --git a/iris-mpc/src/bin/server.rs b/iris-mpc/src/bin/server.rs
@@ -588,7 +588,6 @@ async fn server_main(config: Config) -> eyre::Result<()> {
     tracing::info!("Preparing task monitor");
     let mut background_tasks = TaskMonitor::new();
 
-    // DEBUG: disable heartbeat
     let (tx, rx) = oneshot::channel();
     let _heartbeat = background_tasks.spawn(start_heartbeat(config.party_id, tx));