From 39e752fd405711e3636a192d0a8ab2bdea7e95d7 Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 14:10:31 -0300 Subject: [PATCH 1/7] support liquidated vaults for metrics --- clients/vault/src/metrics.rs | 3 ++- clients/vault/src/system.rs | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/clients/vault/src/metrics.rs b/clients/vault/src/metrics.rs index 05f484a02..1462a8592 100644 --- a/clients/vault/src/metrics.rs +++ b/clients/vault/src/metrics.rs @@ -117,8 +117,9 @@ pub trait VaultDataReader { #[async_trait] impl VaultDataReader for VaultIdManager { + // get_all_entries fetches from active and liquidated vaults async fn get_entries(&self) -> Vec { - self.get_entries().await + self.get_all_entries().await } } diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index c654d54ad..5280b90ca 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -52,6 +52,7 @@ pub struct VaultData { pub vault_id: VaultId, pub stellar_wallet: ArcRwLock, pub metrics: PerCurrencyMetrics, + pub liquidated: bool } #[derive(Clone)] @@ -88,6 +89,7 @@ impl VaultIdManager { vault_id: key.clone(), stellar_wallet: stellar_wallet.clone(), metrics: PerCurrencyMetrics::dummy(), + liquidated: false }, ) }) @@ -95,12 +97,13 @@ impl VaultIdManager { Self { vault_data: Arc::new(RwLock::new(vault_data)), spacewalk_parachain, stellar_wallet } } - async fn add_vault_id(&self, vault_id: VaultId) -> Result<(), Error> { + async fn add_vault_id(&self, vault_id: VaultId, is_liquidated: bool) -> Result<(), Error> { let metrics = PerCurrencyMetrics::new(&vault_id); let data = VaultData { vault_id: vault_id.clone(), stellar_wallet: self.stellar_wallet.clone(), metrics, + liquidated: is_liquidated }; PerCurrencyMetrics::initialize_values(self.spacewalk_parachain.clone(), &data).await; @@ -118,12 +121,17 @@ impl VaultIdManager { .await? { // check if vault is registered + // IDEA 2. Since this is never added to the vault_id_manager, on the metrics (for some of them) + // we will not iterate through it and never add them. match self.spacewalk_parachain.get_vault(&vault_id).await { - Ok(_) => self.add_vault_id(vault_id.clone()).await?, - Err(RuntimeError::VaultLiquidated) => tracing::error!( + Ok(_) => self.add_vault_id(vault_id.clone(), false).await?, + Err(RuntimeError::VaultLiquidated) => { + + self.add_vault_id(vault_id.clone(), true).await?; + tracing::error!( "[{}] Vault is liquidated -- not going to process events for this vault.", - vault_id.pretty_print() - ), + vault_id.pretty_print()); + }, Err(e) => return Err(e.into()), } } @@ -138,7 +146,7 @@ impl VaultIdManager { let vault_id = event.vault_id; if self.spacewalk_parachain.is_this_vault(&vault_id) { tracing::info!("New vault registered: {}", vault_id.pretty_print()); - let _ = self.add_vault_id(vault_id).await; + let _ = self.add_vault_id(vault_id, false).await; } }, |err| tracing::error!("Error (RegisterVaultEvent): {}", err.to_string()), @@ -154,7 +162,15 @@ impl VaultIdManager { self.vault_data.read().await.get(vault_id).cloned() } + // Get all ACTIVE vaults pub async fn get_entries(&self) -> Vec { + self.vault_data.read().await.iter() + .filter(|(_, value)| value.liquidated != true) + .map(|(_, value)| value.clone()).collect() + } + + // Get all vaults including liquidated ones. + pub async fn get_all_entries(&self) -> Vec{ self.vault_data.read().await.iter().map(|(_, value)| value.clone()).collect() } @@ -163,6 +179,7 @@ impl VaultIdManager { .read() .await .iter() + .filter(|(_, value)| value.liquidated != true) .map(|(vault_id, _)| vault_id.clone()) .collect() } From 9f641d2ee13d97522c3a44b2cbf9ab1796acf74c Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 15:17:42 -0300 Subject: [PATCH 2/7] add method to fetch only active vaults ids from manager, only accept requests if not liquidated --- clients/vault/src/replace.rs | 2 +- clients/vault/src/system.rs | 35 +++++++++++++++++-------- pallets/stellar-relay/src/validation.rs | 4 +-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/clients/vault/src/replace.rs b/clients/vault/src/replace.rs index 5c96e8914..b2340837d 100644 --- a/clients/vault/src/replace.rs +++ b/clients/vault/src/replace.rs @@ -35,7 +35,7 @@ pub async fn listen_for_accept_replace( parachain_rpc .on_event::( |event| async move { - let vault = match vault_id_manager.get_vault(&event.old_vault_id).await { + let vault = match vault_id_manager.get_active_vault(&event.old_vault_id).await { Some(x) => x, None => return, // event not directed at this vault }; diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 5280b90ca..7d8c1da19 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -52,7 +52,7 @@ pub struct VaultData { pub vault_id: VaultId, pub stellar_wallet: ArcRwLock, pub metrics: PerCurrencyMetrics, - pub liquidated: bool + pub liquidated: bool, } #[derive(Clone)] @@ -89,7 +89,7 @@ impl VaultIdManager { vault_id: key.clone(), stellar_wallet: stellar_wallet.clone(), metrics: PerCurrencyMetrics::dummy(), - liquidated: false + liquidated: false, }, ) }) @@ -103,7 +103,7 @@ impl VaultIdManager { vault_id: vault_id.clone(), stellar_wallet: self.stellar_wallet.clone(), metrics, - liquidated: is_liquidated + liquidated: is_liquidated, }; PerCurrencyMetrics::initialize_values(self.spacewalk_parachain.clone(), &data).await; @@ -121,16 +121,16 @@ impl VaultIdManager { .await? { // check if vault is registered - // IDEA 2. Since this is never added to the vault_id_manager, on the metrics (for some of them) - // we will not iterate through it and never add them. + // IDEA 2. Since this is never added to the vault_id_manager, on the metrics (for some + // of them) we will not iterate through it and never add them. match self.spacewalk_parachain.get_vault(&vault_id).await { Ok(_) => self.add_vault_id(vault_id.clone(), false).await?, Err(RuntimeError::VaultLiquidated) => { - self.add_vault_id(vault_id.clone(), true).await?; tracing::error!( - "[{}] Vault is liquidated -- not going to process events for this vault.", - vault_id.pretty_print()); + "[{}] Vault is liquidated -- not going to process events for this vault.", + vault_id.pretty_print() + ); }, Err(e) => return Err(e.into()), } @@ -158,19 +158,32 @@ impl VaultIdManager { self.vault_data.read().await.get(vault_id).map(|x| x.stellar_wallet.clone()) } + pub async fn get_active_vault(&self, vault_id: &VaultId) -> Option { + let vault = self.vault_data.read().await.get(vault_id)?.clone(); + // Filter liquidated + if vault.liquidated { + return None; + } + return Some(vault); + } + pub async fn get_vault(&self, vault_id: &VaultId) -> Option { self.vault_data.read().await.get(vault_id).cloned() } // Get all ACTIVE vaults pub async fn get_entries(&self) -> Vec { - self.vault_data.read().await.iter() + self.vault_data + .read() + .await + .iter() .filter(|(_, value)| value.liquidated != true) - .map(|(_, value)| value.clone()).collect() + .map(|(_, value)| value.clone()) + .collect() } // Get all vaults including liquidated ones. - pub async fn get_all_entries(&self) -> Vec{ + pub async fn get_all_entries(&self) -> Vec { self.vault_data.read().await.iter().map(|(_, value)| value.clone()).collect() } diff --git a/pallets/stellar-relay/src/validation.rs b/pallets/stellar-relay/src/validation.rs index cce5b8cf5..783317e34 100644 --- a/pallets/stellar-relay/src/validation.rs +++ b/pallets/stellar-relay/src/validation.rs @@ -92,7 +92,7 @@ fn is_node_id_exist( envelope.statement.slot_index, envelope.statement.node_id ); - return None + return None; } Some(node_id) @@ -174,7 +174,7 @@ pub fn validate_envelopes<'a, T: Config>( for envelope in envelopes { let Some(node_id) = is_node_id_exist::(envelope, validators) else { // ignore this envelope; continue to the next ones - continue + continue; }; // Check if all envelopes are using the same slot index From 0fb784890a5c74445ad08084de5906951b6e288d Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 16:29:15 -0300 Subject: [PATCH 3/7] add metric for liquidated --- clients/vault/src/metrics.rs | 29 +++++++++++++++++++++++++++-- clients/vault/src/replace.rs | 2 +- clients/vault/src/system.rs | 1 + 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/clients/vault/src/metrics.rs b/clients/vault/src/metrics.rs index 1462a8592..bbd3ca1f8 100644 --- a/clients/vault/src/metrics.rs +++ b/clients/vault/src/metrics.rs @@ -9,8 +9,10 @@ use lazy_static::lazy_static; use primitives::{stellar, Asset, DecimalsLookup}; use runtime::{ prometheus::{ - gather, proto::MetricFamily, Encoder, Gauge, GaugeVec, IntCounter, IntGaugeVec, Opts, - Registry, TextEncoder, + core::{AtomicI64, GenericGauge}, + gather, + proto::MetricFamily, + Encoder, Gauge, GaugeVec, IntCounter, IntGaugeVec, Opts, Registry, TextEncoder, }, types::currency_id::CurrencyIdExt, AggregateUpdatedEvent, CollateralBalancesPallet, CurrencyId, Error as RuntimeError, FixedU128, @@ -83,6 +85,11 @@ lazy_static! { pub static ref RESTART_COUNT: IntCounter = IntCounter::new("restart_count", "Number of service restarts") .expect("Failed to create prometheus metric"); + pub static ref LIQUIDATED: IntGaugeVec = IntGaugeVec::new( + Opts::new("liquidated", "Boolean reporting if the vault is currently liquidated"), + &[CURRENCY_LABEL] + ) + .expect("Failed to create prometheus metric"); } const STELLAR_NATIVE_ASSET_TYPE: [u8; 6] = *b"native"; @@ -108,11 +115,14 @@ pub struct PerCurrencyMetrics { asset_balance: XLMBalance, issues: RequestCounter, redeems: RequestCounter, + liquidated: GenericGauge, } #[async_trait] pub trait VaultDataReader { async fn get_entries(&self) -> Vec; + + async fn get_vault(&self, vault_id: &VaultId) -> Option; } #[async_trait] @@ -121,6 +131,9 @@ impl VaultDataReader for VaultIdManager { async fn get_entries(&self) -> Vec { self.get_all_entries().await } + async fn get_vault(&self, vault_id: &VaultId) -> Option { + self.get_vault(vault_id) + } } struct DisplayLabels { @@ -232,6 +245,7 @@ impl PerCurrencyMetrics { completed_count: REDEEMS.with(&request_type_label("completed")), expired_count: REDEEMS.with(&request_type_label("expired")), }, + liquidated: LIQUIDATED.with(&labels), } } @@ -260,6 +274,7 @@ pub fn register_custom_metrics() -> Result<(), RuntimeError> { REGISTRY.register(Box::new(MEAN_POLL_DURATION.clone()))?; REGISTRY.register(Box::new(MEAN_SCHEDULED_DURATION.clone()))?; REGISTRY.register(Box::new(RESTART_COUNT.clone()))?; + REGISTRY.register(Box::new(LIQUIDATED.clone()))?; Ok(()) } @@ -482,6 +497,15 @@ async fn publish_redeem_count( } } +async fn update_liquidation_status(vault: &VaultData, vault_id_manager: &V) { + let maybe_vault_data = vault_id_manager.get_vault(&vault.vault_id).await; + + if let Some(vault_data) = maybe_vault_data { + let liquidated_flag: i64 = if vault_data.liquidated == true { 1 } else { 0 }; + vault.metrics.liquidated.set(liquidated_flag); + } +} + pub async fn monitor_bridge_metrics( parachain_rpc: SpacewalkParachain, vault_id_manager: VaultIdManager, @@ -500,6 +524,7 @@ pub async fn monitor_bridge_metrics( .iter() .filter(|vault| vault.vault_id.collateral_currency() == **currency_id) { + let _ = update_liquidation_status(vault, vault_id_manager).await; let _ = publish_locked_collateral(vault, parachain_rpc.clone()).await; let _ = publish_required_collateral(vault, parachain_rpc.clone()).await; publish_collateralization(vault, parachain_rpc.clone()).await; diff --git a/clients/vault/src/replace.rs b/clients/vault/src/replace.rs index b2340837d..5c96e8914 100644 --- a/clients/vault/src/replace.rs +++ b/clients/vault/src/replace.rs @@ -35,7 +35,7 @@ pub async fn listen_for_accept_replace( parachain_rpc .on_event::( |event| async move { - let vault = match vault_id_manager.get_active_vault(&event.old_vault_id).await { + let vault = match vault_id_manager.get_vault(&event.old_vault_id).await { Some(x) => x, None => return, // event not directed at this vault }; diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 7d8c1da19..a3458fd3c 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -204,6 +204,7 @@ impl VaultIdManager { .read() .await .iter() + .filter(|(_, value)| value.liquidated != true) .map(|(vault_id, data)| (vault_id.clone(), data.stellar_wallet.clone())) .collect() } From dc06c66b5c7aa816e0b91176cbd0d332d959d78a Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 16:37:08 -0300 Subject: [PATCH 4/7] clippy fixes --- clients/secretstellartest.txt | 1 + clients/vault/src/metrics.rs | 2 +- clients/vault/src/system.rs | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 clients/secretstellartest.txt diff --git a/clients/secretstellartest.txt b/clients/secretstellartest.txt new file mode 100644 index 000000000..aa042f281 --- /dev/null +++ b/clients/secretstellartest.txt @@ -0,0 +1 @@ +SB6WHKIU2HGVBRNKNOEOQUY4GFC4ZLG5XPGWLEAHTIZXBXXYACC76VSQ \ No newline at end of file diff --git a/clients/vault/src/metrics.rs b/clients/vault/src/metrics.rs index bbd3ca1f8..b14e145e2 100644 --- a/clients/vault/src/metrics.rs +++ b/clients/vault/src/metrics.rs @@ -132,7 +132,7 @@ impl VaultDataReader for VaultIdManager { self.get_all_entries().await } async fn get_vault(&self, vault_id: &VaultId) -> Option { - self.get_vault(vault_id) + self.get_vault(vault_id).await } } diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index a3458fd3c..22597c2ed 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -177,7 +177,7 @@ impl VaultIdManager { .read() .await .iter() - .filter(|(_, value)| value.liquidated != true) + .filter(|(_, value)| !value.liquidated) .map(|(_, value)| value.clone()) .collect() } @@ -192,7 +192,7 @@ impl VaultIdManager { .read() .await .iter() - .filter(|(_, value)| value.liquidated != true) + .filter(|(_, value)| !value.liquidated) .map(|(vault_id, _)| vault_id.clone()) .collect() } @@ -204,7 +204,7 @@ impl VaultIdManager { .read() .await .iter() - .filter(|(_, value)| value.liquidated != true) + .filter(|(_, value)| !value.liquidated) .map(|(vault_id, data)| (vault_id.clone(), data.stellar_wallet.clone())) .collect() } From a1a7c9bcceb0e0177214e2b2472f99299913bab2 Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 17:39:00 -0300 Subject: [PATCH 5/7] simplify liquidation status metric updater --- clients/secretstellartest.txt | 1 - clients/vault/src/metrics.rs | 13 +++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 clients/secretstellartest.txt diff --git a/clients/secretstellartest.txt b/clients/secretstellartest.txt deleted file mode 100644 index aa042f281..000000000 --- a/clients/secretstellartest.txt +++ /dev/null @@ -1 +0,0 @@ -SB6WHKIU2HGVBRNKNOEOQUY4GFC4ZLG5XPGWLEAHTIZXBXXYACC76VSQ \ No newline at end of file diff --git a/clients/vault/src/metrics.rs b/clients/vault/src/metrics.rs index b14e145e2..50c85f705 100644 --- a/clients/vault/src/metrics.rs +++ b/clients/vault/src/metrics.rs @@ -257,6 +257,7 @@ impl PerCurrencyMetrics { publish_locked_collateral(vault, parachain_rpc.clone()), publish_required_collateral(vault, parachain_rpc.clone()), publish_collateralization(vault, parachain_rpc.clone()), + update_liquidation_status(vault) ); } } @@ -497,13 +498,9 @@ async fn publish_redeem_count( } } -async fn update_liquidation_status(vault: &VaultData, vault_id_manager: &V) { - let maybe_vault_data = vault_id_manager.get_vault(&vault.vault_id).await; - - if let Some(vault_data) = maybe_vault_data { - let liquidated_flag: i64 = if vault_data.liquidated == true { 1 } else { 0 }; - vault.metrics.liquidated.set(liquidated_flag); - } +async fn update_liquidation_status(vault: &VaultData) { + let liquidated_flag: i64 = if vault.liquidated { 1 } else { 0 }; + vault.metrics.liquidated.set(liquidated_flag); } pub async fn monitor_bridge_metrics( @@ -524,7 +521,7 @@ pub async fn monitor_bridge_metrics( .iter() .filter(|vault| vault.vault_id.collateral_currency() == **currency_id) { - let _ = update_liquidation_status(vault, vault_id_manager).await; + let _ = update_liquidation_status(vault).await; let _ = publish_locked_collateral(vault, parachain_rpc.clone()).await; let _ = publish_required_collateral(vault, parachain_rpc.clone()).await; publish_collateralization(vault, parachain_rpc.clone()).await; From 49bfc01f5539b976e2c7bf86ffc4a519d7d7c7bd Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 17:45:54 -0300 Subject: [PATCH 6/7] remove comment --- clients/vault/src/system.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 22597c2ed..cab45adcc 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -121,8 +121,6 @@ impl VaultIdManager { .await? { // check if vault is registered - // IDEA 2. Since this is never added to the vault_id_manager, on the metrics (for some - // of them) we will not iterate through it and never add them. match self.spacewalk_parachain.get_vault(&vault_id).await { Ok(_) => self.add_vault_id(vault_id.clone(), false).await?, Err(RuntimeError::VaultLiquidated) => { From 8a0f7319f305bf83c3c199834fe28f9b003bf329 Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Wed, 16 Oct 2024 17:47:15 -0300 Subject: [PATCH 7/7] remove unused method --- clients/vault/src/metrics.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clients/vault/src/metrics.rs b/clients/vault/src/metrics.rs index 50c85f705..9209647ff 100644 --- a/clients/vault/src/metrics.rs +++ b/clients/vault/src/metrics.rs @@ -121,8 +121,6 @@ pub struct PerCurrencyMetrics { #[async_trait] pub trait VaultDataReader { async fn get_entries(&self) -> Vec; - - async fn get_vault(&self, vault_id: &VaultId) -> Option; } #[async_trait] @@ -131,9 +129,6 @@ impl VaultDataReader for VaultIdManager { async fn get_entries(&self) -> Vec { self.get_all_entries().await } - async fn get_vault(&self, vault_id: &VaultId) -> Option { - self.get_vault(vault_id).await - } } struct DisplayLabels {