feat(cat-gateway): Add Chain Follower metrics to improve observability

catalyst-gateway/bin/Cargo.toml

@@ -53,7 +53,7 @@ local-ip-address = "0.6.3"
 gethostname = "0.5.0"
 hex = "0.4.3"
 handlebars = "6.2.0"
-anyhow = "1.0.92"
+anyhow = "1.0.95"
 duration-string = "0.4.0"
 build-info = "0.0.39"
 ed25519-dalek = "2.1.1"

catalyst-gateway/bin/src/metrics/chain_follower.rs

@@ -1 +1,116 @@
 //! Metrics related to Chain Follower analytics.
+
+use std::{
+    sync::atomic::{AtomicBool, Ordering},
+    thread,
+};
+
+use cardano_chain_follower::Statistics;
+
+use crate::settings::Settings;
+
+/// This is to prevent the init function from accidentally being called multiple times.
+static IS_INITIALIZED: AtomicBool = AtomicBool::new(false);
+
+/// Starts a background thread to periodically update Chain Follower metrics.
+///
+/// This function spawns a thread that updates the Chain Follower metrics
+/// at regular intervals defined by `METRICS_FOLLOWER_INTERVAL`.
+pub(crate) fn init_metrics_reporter() {
+    if IS_INITIALIZED.swap(true, Ordering::SeqCst) {
+        return;
+    }
+
+    let api_host_names = Settings::api_host_names().join(",");
+    let service_id = Settings::service_id();
+    let chain = Settings::cardano_network();
+
+    thread::spawn(move || {
+        loop {
+            {
+                let follower_stats = Statistics::new(chain);
+
+                // mithril part
+                let is_downloading =
+                    follower_stats.mithril.dl_end < follower_stats.mithril.dl_start;
+                let is_extracting =
+                    follower_stats.mithril.extract_end < follower_stats.mithril.extract_start;
+                let is_validating =
+                    follower_stats.mithril.validate_end < follower_stats.mithril.validate_start;
+
+                if is_downloading {
+                    reporter::FOLLOWER_DL_FAILURES
+                        .with_label_values(&[&api_host_names, service_id])
+                        .set(i64::try_from(follower_stats.mithril.dl_failures).unwrap_or(-1));
+                    reporter::FOLLOWER_DL_SIZE
+                        .with_label_values(&[&api_host_names, service_id])
+                        .set(i64::try_from(follower_stats.mithril.dl_size).unwrap_or(-1));
+                }
+                if is_extracting {
+                    reporter::FOLLOWER_EXTRACT_FAILURES
+                        .with_label_values(&[&api_host_names, service_id])
+                        .set(i64::try_from(follower_stats.mithril.extract_failures).unwrap_or(-1));
+                    reporter::FOLLOWER_EXTRACT_SIZE
+                        .with_label_values(&[&api_host_names, service_id])
+                        .set(i64::try_from(follower_stats.mithril.extract_size).unwrap_or(-1));
+                }
+                if is_validating {}
+
+                // live part
+            }
+
+            thread::sleep(Settings::metrics_follower_interval());
+        }
+    });
+}
+
+/// All the related Chain Follower reporting metrics to the Prometheus service are inside
+/// this module.
+mod reporter {
+    use std::sync::LazyLock;
+
+    use prometheus::{register_int_gauge_vec, IntGaugeVec};
+
+    /// Labels for the chain follower metrics
+    const FOLLOWER_METRIC_LABELS: [&str; 2] = ["api_host_names", "service_id"];
+
+    /// The number of times download failed.
+    pub(super) static FOLLOWER_DL_FAILURES: LazyLock<IntGaugeVec> = LazyLock::new(|| {
+        register_int_gauge_vec!(
+            "follower_dl_failures",
+            "Number of times download failed",
+            &FOLLOWER_METRIC_LABELS
+        )
+        .unwrap()
+    });
+
+    /// The size of the download archive, in bytes.
+    pub(super) static FOLLOWER_DL_SIZE: LazyLock<IntGaugeVec> = LazyLock::new(|| {
+        register_int_gauge_vec!(
+            "follower_dl_size",
+            "Size of the download archive",
+            &FOLLOWER_METRIC_LABELS
+        )
+        .unwrap()
+    });
+
+    /// The number of times extraction failed.
+    pub(super) static FOLLOWER_EXTRACT_FAILURES: LazyLock<IntGaugeVec> = LazyLock::new(|| {
+        register_int_gauge_vec!(
+            "follower_extract_failures",
+            "Number of times extraction failed",
+            &FOLLOWER_METRIC_LABELS
+        )
+        .unwrap()
+    });
+
+    /// The size of last extracted snapshot.
+    pub(super) static FOLLOWER_EXTRACT_SIZE: LazyLock<IntGaugeVec> = LazyLock::new(|| {
+        register_int_gauge_vec!(
+            "follower_extract_size",
+            "Size of last extracted snapshot",
+            &FOLLOWER_METRIC_LABELS
+        )
+        .unwrap()
+    });
+}

catalyst-gateway/bin/src/metrics/memory.rs

@@ -22,9 +22,9 @@ static IS_INITIALIZED: AtomicBool = AtomicBool::new(false);
 
 /// Starts a background thread to periodically update memory metrics.
 ///
-/// This function spawns a thread that updates the global `MemoryMetrics`
-/// structure at regular intervals defined by `UPDATE_INTERVAL_MILLI`.
-pub(crate) fn init_metrics_updater() {
+/// This function spawns a thread that updates the memory metrics
+/// at regular intervals defined by `METRICS_MEMORY_INTERVAL`.
+pub(crate) fn init_metrics_reporter() {
     if IS_INITIALIZED.swap(true, Ordering::SeqCst) {
         return;
     }
@@ -82,7 +82,7 @@ mod reporter {
 
     use prometheus::{register_int_gauge_vec, IntGaugeVec};
 
-    /// Labels for the client metrics
+    /// Labels for the memory metrics
     const MEMORY_METRIC_LABELS: [&str; 2] = ["api_host_names", "service_id"];
 
     /// The "physical" memory used by this process, in bytes.

catalyst-gateway/bin/src/metrics/mod.rs

@@ -14,7 +14,8 @@ pub(crate) mod memory;
 /// Returns the default prometheus registry.
 #[must_use]
 pub(crate) fn init_prometheus() -> Registry {
-    memory::init_metrics_updater();
+    chain_follower::init_metrics_reporter();
+    memory::init_metrics_reporter();
 
     default_registry().clone()
 }

catalyst-gateway/bin/src/settings/mod.rs

@@ -54,6 +54,9 @@ const CHECK_CONFIG_TICK_DEFAULT: &str = "5s";
 /// Default `METRICS_MEMORY_INTERVAL`.
 const METRICS_MEMORY_INTERVAL_DEFAULT: &str = "1s";
 
+/// Default `METRICS_FOLLOWER_INTERVAL`.
+const METRICS_FOLLOWER_INTERVAL_DEFAULT: &str = "1s";
+
 /// Default Event DB URL.
 const EVENT_DB_URL_DEFAULT: &str =
     "postgresql://postgres:postgres@localhost/catalyst_events?sslmode=disable";
@@ -149,6 +152,9 @@ struct EnvVars {
 
     /// Interval for updating and sending memory metrics.
     metrics_memory_interval: Duration,
+
+    /// Interval for updating and sending Chain Follower metrics.
+    metrics_follower_interval: Duration,
 }
 
 // Lazy initialization of all env vars which are not command line parameters.
@@ -194,6 +200,10 @@ static ENV_VARS: LazyLock<EnvVars> = LazyLock::new(|| {
             "METRICS_MEMORY_INTERVAL",
             METRICS_MEMORY_INTERVAL_DEFAULT,
         ),
+        metrics_follower_interval: StringEnvVar::new_as_duration(
+            "METRICS_FOLLOWER_INTERVAL",
+            METRICS_FOLLOWER_INTERVAL_DEFAULT,
+        ),
     }
 });
 
@@ -292,6 +302,11 @@ impl Settings {
         ENV_VARS.metrics_memory_interval
     }
 
+    /// The Chain Follower metrics interval
+    pub(crate) fn metrics_follower_interval() -> Duration {
+        ENV_VARS.metrics_follower_interval
+    }
+
     /// Get a list of all host names to serve the API on.
     ///
     /// Used by the `OpenAPI` Documentation to point to the correct backend.