move memory benchmark behind a flag

worldcoin · Jan 21, 2025 · ff7e3ca · ff7e3ca
1 parent c85c402
commit ff7e3ca
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 20 deletions.
diff --git a/deploy/stage/common-values-iris-mpc.yaml b/deploy/stage/common-values-iris-mpc.yaml
@@ -1,4 +1,4 @@
-image: "ghcr.io/worldcoin/iris-mpc:86e7a6c76cf6888d268d214a2ca73226f8fdd426"
+image: "ghcr.io/worldcoin/iris-mpc:f6b975ac741268aa143c308022ce2a8011c8b95b"
 
 environment: stage
 replicaCount: 1

diff --git a/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml
@@ -78,7 +78,7 @@ env:
     value: "even_odd_binary_output_16k"
 
   - name: SMPC__LOAD_CHUNKS_PARALLELISM
-    value: "64"
+    value: "256"
 
   - name: SMPC__LOAD_CHUNKS_DEFAULT_CLIENT
     value: "false"

diff --git a/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml
@@ -1,6 +1,6 @@
 env:
   - name: RUST_LOG
-    value: "debug"
+    value: "info"
 
   - name: RUST_BACKTRACE
     value: "full"
@@ -78,7 +78,7 @@ env:
     value: "even_odd_binary_output_16k"
 
   - name: SMPC__LOAD_CHUNKS_PARALLELISM
-    value: "64"
+    value: "256"
 
   - name: SMPC__LOAD_CHUNKS_DEFAULT_CLIENT
     value: "false"

diff --git a/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml
@@ -78,7 +78,7 @@ env:
     value: "even_odd_binary_output_16k"
 
   - name: SMPC__LOAD_CHUNKS_PARALLELISM
-    value: "64"
+    value: "256"
 
   - name: SMPC__LOAD_CHUNKS_DEFAULT_CLIENT
     value: "false"

diff --git a/iris-mpc-common/src/config/mod.rs b/iris-mpc-common/src/config/mod.rs
@@ -120,6 +120,9 @@ pub struct Config {
 
     #[serde(default)]
     pub load_chunks_static_ips: usize,
+
+    #[serde(default)]
+    pub test_load_into_memory: bool,
 }
 
 fn default_load_chunks_parallelism() -> usize {

diff --git a/iris-mpc-store/src/s3_importer.rs b/iris-mpc-store/src/s3_importer.rs
@@ -334,7 +334,7 @@ pub async fn fetch_to_memory(
                     match result.read_exact(&mut slice).await {
                         Ok(_) => {
                             let iris = S3StoredIris::from_bytes(&slice)?;
-                            if iris.id % 10000 == 0 {
+                            if iris.id % 100000 == 0 {
                                 tracing::info!("Fetched iris with id: {}", iris.id);
                             }
                         }

diff --git a/iris-mpc/src/bin/server.rs b/iris-mpc/src/bin/server.rs
@@ -1181,20 +1181,23 @@ async fn server_main(config: Config) -> eyre::Result<()> {
                             db_chunks_folder_name.clone(),
                         )
                         .await?;
-                        let fetch_test_ts = Instant::now();
-                        fetch_to_memory(
-                            Arc::new(db_chunks_s3_store),
-                            load_chunks_parallelism,
-                            db_chunks_folder_name.clone(),
-                            last_snapshot_details.clone(),
-                        )
-                        .await?;
-                        let elapsed = fetch_test_ts.elapsed();
-                        tracing::info!(
-                            "Fetch to memory took {:?} with {} par",
-                            elapsed,
-                            load_chunks_parallelism
-                        );
+
+                        if config.test_load_into_memory {
+                            let fetch_test_ts = Instant::now();
+                            fetch_to_memory(
+                                Arc::new(db_chunks_s3_store),
+                                load_chunks_parallelism,
+                                db_chunks_folder_name.clone(),
+                                last_snapshot_details.clone(),
+                            )
+                            .await?;
+                            let elapsed = fetch_test_ts.elapsed();
+                            tracing::info!(
+                                "Fetch to memory took {:?} with {} par",
+                                elapsed,
+                                load_chunks_parallelism
+                            );
+                        }
 
                         let s3_store = S3Store::new(db_chunks_s3_client, db_chunks_bucket_name);
                         let s3_arc = Arc::new(s3_store);