From cc22ea8600dcf6fd165f365bdbcd1852a91cbed6 Mon Sep 17 00:00:00 2001 From: Ertugrul Aypek Date: Mon, 9 Dec 2024 21:27:49 +0100 Subject: [PATCH] change folder name --- deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml | 3 +++ deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml | 3 +++ deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml | 3 +++ iris-mpc-common/src/config/mod.rs | 3 +++ iris-mpc-store/src/lib.rs | 2 +- iris-mpc-store/src/s3_importer.rs | 11 ++++++++--- iris-mpc/src/bin/server.rs | 9 ++++++++- 7 files changed, 29 insertions(+), 5 deletions(-) diff --git a/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml index efaa0d52d..d77928e19 100644 --- a/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml @@ -71,6 +71,9 @@ env: - name: SMPC__DB_CHUNKS_BUCKET_NAME value: "iris-mpc-db-exporter-store-node-0-stage-eu-north-1" + - name: SMPC__DB_CHUNKS_FOLDER_NAME + value: "binary_output_4k" + - name: SMPC__CLEAR_DB_BEFORE_INIT value: "true" diff --git a/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml index 81524f5f2..ddf1476e9 100644 --- a/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml @@ -71,6 +71,9 @@ env: - name: SMPC__DB_CHUNKS_BUCKET_NAME value: "iris-mpc-db-exporter-store-node-1-stage-eu-north-1" + - name: SMPC__DB_CHUNKS_FOLDER_NAME + value: "binary_output_4k" + - name: SMPC__CLEAR_DB_BEFORE_INIT value: "true" diff --git a/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml index e4f1a44c5..c16ff6b80 100644 --- a/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml @@ -71,6 +71,9 @@ env: - name: SMPC__DB_CHUNKS_BUCKET_NAME value: "iris-mpc-db-exporter-store-node-2-stage-eu-north-1" + - name: SMPC__DB_CHUNKS_FOLDER_NAME + value: "binary_output_4k" + - name: SMPC__CLEAR_DB_BEFORE_INIT value: "true" diff --git a/iris-mpc-common/src/config/mod.rs b/iris-mpc-common/src/config/mod.rs index 2f84f8d61..45478fec6 100644 --- a/iris-mpc-common/src/config/mod.rs +++ b/iris-mpc-common/src/config/mod.rs @@ -99,6 +99,9 @@ pub struct Config { /// updated during the DB export to S3 #[serde(default = "default_db_load_safety_overlap_seconds")] pub db_load_safety_overlap_seconds: i64, + + #[serde(default)] + pub db_chunks_folder_name: String, } fn default_load_chunks_parallelism() -> usize { diff --git a/iris-mpc-store/src/lib.rs b/iris-mpc-store/src/lib.rs index fceef816a..af1486764 100644 --- a/iris-mpc-store/src/lib.rs +++ b/iris-mpc-store/src/lib.rs @@ -18,7 +18,7 @@ pub use s3_importer::{fetch_and_parse_chunks, last_snapshot_timestamp, ObjectSto use sqlx::{ migrate::Migrator, postgres::PgPoolOptions, Executor, PgPool, Postgres, Row, Transaction, }; -use std::{ops::DerefMut, pin::Pin, u32}; +use std::{ops::DerefMut, pin::Pin}; const APP_NAME: &str = "SMPC"; const MAX_CONNECTIONS: u32 = 100; diff --git a/iris-mpc-store/src/s3_importer.rs b/iris-mpc-store/src/s3_importer.rs index defabdd42..66ba7203f 100644 --- a/iris-mpc-store/src/s3_importer.rs +++ b/iris-mpc-store/src/s3_importer.rs @@ -72,15 +72,20 @@ impl ObjectStore for S3Store { } } -pub async fn last_snapshot_timestamp(store: &impl ObjectStore) -> eyre::Result { +pub async fn last_snapshot_timestamp( + store: &impl ObjectStore, + folder_name: String, +) -> eyre::Result { + tracing::info!("Looking for last snapshot time in folder: {}", folder_name); + let start_path = format!("{}/", folder_name); store .list_objects() .await? .into_iter() - .filter(|f| f.starts_with("output/") && f.ends_with(".timestamp")) + .filter(|f| f.starts_with(start_path.as_str()) && f.ends_with(".timestamp")) .filter_map(|f| { f.replace(".timestamp", "") - .replace("output/", "") + .replace(start_path.as_str(), "") .parse::() .ok() }) diff --git a/iris-mpc/src/bin/server.rs b/iris-mpc/src/bin/server.rs index 67f64611c..7e4bab23b 100644 --- a/iris-mpc/src/bin/server.rs +++ b/iris-mpc/src/bin/server.rs @@ -903,6 +903,7 @@ async fn server_main(config: Config) -> eyre::Result<()> { let load_chunks_parallelism = config.load_chunks_parallelism; let db_chunks_bucket_name = config.db_chunks_bucket_name.clone(); + let db_chunks_folder_name = config.db_chunks_folder_name.clone(); let (tx, rx) = oneshot::channel(); background_tasks.spawn_blocking(move || { @@ -984,9 +985,15 @@ async fn server_main(config: Config) -> eyre::Result<()> { let s3_store = S3Store::new(s3_client_clone, db_chunks_bucket_name); tokio::runtime::Handle::current().block_on(async { // First fetch last snapshot from S3 - let last_snapshot_timestamp = last_snapshot_timestamp(&s3_store).await?; + let last_snapshot_timestamp = + last_snapshot_timestamp(&s3_store, db_chunks_folder_name).await?; let min_last_modified_at = last_snapshot_timestamp - config.db_load_safety_overlap_seconds; + tracing::info!( + "Last snapshot timestamp: {}, min_last_modified_at: {}", + last_snapshot_timestamp, + min_last_modified_at + ); let stream_s3 = fetch_and_parse_chunks(&s3_store, load_chunks_parallelism) .await .map(|result| result.map(IrisSource::S3))