increase ib timeout

worldcoin · Nov 1, 2024 · 1a7666d · 1a7666d
1 parent 3eb1acb
commit 1a7666d
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 0 deletions.
diff --git a/deploy/prod/smpcv2-0-prod/values-iris-mpc.yaml b/deploy/prod/smpcv2-0-prod/values-iris-mpc.yaml
@@ -11,6 +11,9 @@ env:
   - name: FI_EFA_USE_DEVICE_RDMA
     value: "1"
 
+  - name: NCCL_IB_TIMEOUT
+    value: "30"
+
   - name: NCCL_COMM_ID
     value: "iris-mpc-node.1.smpcv2.worldcoin.org:4000"
 

diff --git a/deploy/prod/smpcv2-1-prod/values-iris-mpc.yaml b/deploy/prod/smpcv2-1-prod/values-iris-mpc.yaml
@@ -14,6 +14,9 @@ env:
   - name: FI_EFA_USE_DEVICE_RDMA
     value: "1"
 
+  - name: NCCL_IB_TIMEOUT
+    value: "30"
+
   - name: NCCL_COMM_ID
     value: "iris-mpc-node.1.smpcv2.worldcoin.org:4000"
 

diff --git a/deploy/prod/smpcv2-2-prod/values-iris-mpc.yaml b/deploy/prod/smpcv2-2-prod/values-iris-mpc.yaml
@@ -14,6 +14,9 @@ env:
   - name: FI_EFA_USE_DEVICE_RDMA
     value: "1"
 
+  - name: NCCL_IB_TIMEOUT
+    value: "30"
+
   - name: NCCL_COMM_ID
     value: "iris-mpc-node.1.smpcv2.worldcoin.org:4000"