Merge branch 'main' into test/nightly-schemathesis

.config/dictionaries/project.dic

@@ -351,3 +351,5 @@ xpub
 xpublic
 xvfb
 yoroi
+stake1u94ullc9nj9gawc08990nx8hwgw80l9zpmr8re44kydqy9cdjq6rq
+invalid1u9nlq5nmuzthw3vhgakfpxyq4r0zl2c0p8uqy24gpyjsa6c3df4h6

catalyst-gateway/bin/src/db/index/queries/cql/get_all_stake_addrs.cql

@@ -0,0 +1,3 @@
+SELECT 
+    stake_address,vote_key
+FROM cip36_registration;

catalyst-gateway/bin/src/db/index/queries/mod.rs

@@ -16,6 +16,7 @@ use rbac::{
     get_role0_chain_root::GetRole0ChainRootQuery,
 };
 use registrations::{
+    get_all_stakes_and_vote_keys::GetAllStakesAndVoteKeysQuery,
     get_from_stake_addr::GetRegistrationQuery, get_from_stake_hash::GetStakeAddrQuery,
     get_from_vote_key::GetStakeAddrFromVoteKeyQuery, get_invalid::GetInvalidRegistrationQuery,
 };
@@ -96,6 +97,8 @@ pub(crate) enum PreparedSelectQuery {
     RegistrationsByChainRoot,
     /// Get chain root by role0 key
     ChainRootByRole0Key,
+    /// Get all stake and vote keys for snapshot (`stake_pub_key,vote_key`)
+    GetAllStakesAndVoteKeys,
 }
 
 /// All prepared UPSERT query statements (inserts/updates a single value of data).
@@ -157,6 +160,8 @@ pub(crate) struct PreparedQueries {
     registrations_by_chain_root_query: PreparedStatement,
     /// Get chain root by role0 key
     chain_root_by_role0_key_query: PreparedStatement,
+    /// Get all stake and vote keys (`stake_key,vote_key`) for snapshot
+    get_all_stakes_and_vote_keys_query: PreparedStatement,
 }
 
 /// An individual query response that can fail
@@ -193,7 +198,9 @@ impl PreparedQueries {
         let chain_root_by_stake_address = GetChainRootQuery::prepare(session.clone()).await;
         let registrations_by_chain_root =
             GetRegistrationsByChainRootQuery::prepare(session.clone()).await;
-        let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session).await;
+        let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session.clone()).await;
+        let get_all_stakes_and_vote_keys_query =
+            GetAllStakesAndVoteKeysQuery::prepare(session).await;
 
         let (
             txo_insert_queries,
@@ -241,6 +248,7 @@ impl PreparedQueries {
             chain_root_by_stake_address_query: chain_root_by_stake_address?,
             registrations_by_chain_root_query: registrations_by_chain_root?,
             chain_root_by_role0_key_query: chain_root_by_role0_key?,
+            get_all_stakes_and_vote_keys_query: get_all_stakes_and_vote_keys_query?,
         })
     }
 
@@ -331,6 +339,9 @@ impl PreparedQueries {
                 &self.registrations_by_chain_root_query
             },
             PreparedSelectQuery::ChainRootByRole0Key => &self.chain_root_by_role0_key_query,
+            PreparedSelectQuery::GetAllStakesAndVoteKeys => {
+                &self.get_all_stakes_and_vote_keys_query
+            },
         };
 
         session

catalyst-gateway/bin/src/db/index/queries/registrations/get_all_stakes_and_vote_keys.rs

@@ -0,0 +1,61 @@
+//! Get all stake and vote keys (`stake_pub_key,vote_key`)
+//! Result is used to compose various query registrations for snapshot.
+
+use std::sync::Arc;
+
+use scylla::{
+    prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, DeserializeRow,
+    SerializeRow, Session,
+};
+use tracing::error;
+
+use crate::db::index::{
+    queries::{PreparedQueries, PreparedSelectQuery},
+    session::CassandraSession,
+};
+
+/// Get all (`stake_addr,vote` keys)
+/// [(`stake_addr,vote_key`)]
+const GET_ALL_STAKES_AND_VOTE_KEYS: &str = include_str!("../cql/get_all_stake_addrs.cql");
+
+/// Get all stake and vote keys from cip36 registration
+#[derive(SerializeRow)]
+pub(crate) struct GetAllStakesAndVoteKeysParams {}
+
+/// Get stakes and vote keys for snapshot.
+#[derive(DeserializeRow)]
+pub(crate) struct GetAllStakesAndVoteKeysQuery {
+    /// Full Stake Address (not hashed, 32 byte ED25519 Public key).
+    pub stake_address: Vec<u8>,
+    /// Voting Public Key
+    pub vote_key: Vec<u8>,
+}
+
+impl GetAllStakesAndVoteKeysQuery {
+    /// Prepares get all `stake_addr` paired with vote keys [(`stake_addr,vote_key`)]
+    pub(crate) async fn prepare(session: Arc<Session>) -> anyhow::Result<PreparedStatement> {
+        let get_all_stake_and_vote_keys = PreparedQueries::prepare(
+            session,
+            GET_ALL_STAKES_AND_VOTE_KEYS,
+            scylla::statement::Consistency::All,
+            true,
+        )
+        .await;
+
+        get_all_stake_and_vote_keys.inspect_err(
+            |error| error!(error=%error, "Failed to prepare get all (stake addrs, vote_keys)"),
+        )
+    }
+
+    /// Executes get all `stake_addr` paired with vote keys [(`stake_addr,vote_key`)]
+    pub(crate) async fn execute(
+        session: &CassandraSession, params: GetAllStakesAndVoteKeysParams,
+    ) -> anyhow::Result<TypedRowStream<GetAllStakesAndVoteKeysQuery>> {
+        let iter = session
+            .execute_iter(PreparedSelectQuery::GetAllStakesAndVoteKeys, params)
+            .await?
+            .rows_stream::<GetAllStakesAndVoteKeysQuery>()?;
+
+        Ok(iter)
+    }
+}

catalyst-gateway/bin/src/db/index/queries/registrations/get_from_stake_addr.rs

@@ -24,10 +24,10 @@ pub(crate) struct GetRegistrationParams {
     pub stake_address: Vec<u8>,
 }
 
-impl From<&ed25519_dalek::VerifyingKey> for GetRegistrationParams {
-    fn from(value: &ed25519_dalek::VerifyingKey) -> Self {
+impl From<Vec<u8>> for GetRegistrationParams {
+    fn from(value: Vec<u8>) -> Self {
         GetRegistrationParams {
-            stake_address: value.as_bytes().to_vec(),
+            stake_address: value,
         }
     }
 }

catalyst-gateway/bin/src/db/index/queries/registrations/get_invalid.rs

@@ -8,9 +8,12 @@ use scylla::{
 };
 use tracing::error;
 
-use crate::db::index::{
-    queries::{PreparedQueries, PreparedSelectQuery},
-    session::CassandraSession,
+use crate::{
+    db::index::{
+        queries::{PreparedQueries, PreparedSelectQuery},
+        session::CassandraSession,
+    },
+    service::common::types::cardano::slot_no::SlotNo,
 };
 
 /// Get invalid registrations from stake addr query.
@@ -28,12 +31,10 @@ pub(crate) struct GetInvalidRegistrationParams {
 
 impl GetInvalidRegistrationParams {
     /// Create a new instance of [`GetInvalidRegistrationParams`]
-    pub(crate) fn new(
-        stake_address: Vec<u8>, slot_no: num_bigint::BigInt,
-    ) -> GetInvalidRegistrationParams {
+    pub(crate) fn new(stake_address: Vec<u8>, slot_no: SlotNo) -> GetInvalidRegistrationParams {
         Self {
             stake_address,
-            slot_no,
+            slot_no: slot_no.into(),
         }
     }
 }

catalyst-gateway/bin/src/db/index/queries/registrations/mod.rs

@@ -1,4 +1,5 @@
 //! Registration related queries.
+pub(crate) mod get_all_stakes_and_vote_keys;
 pub(crate) mod get_from_stake_addr;
 pub(crate) mod get_from_stake_hash;
 pub(crate) mod get_from_vote_key;

catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql

@@ -1,5 +1,5 @@
 -- Index of CIP-36 registrations searchable by Stake Address.
--- Full registration data needs to be queried from the man cip36 registration tables.
+-- Full registration data needs to be queried from the main cip36 registration tables.
 -- Includes both Valid and Invalid registrations.
 CREATE TABLE IF NOT EXISTS cip36_registration_for_vote_key (
     -- Primary Key Data

catalyst-gateway/bin/src/db/index/tests/scylla_queries.rs

@@ -4,16 +4,19 @@
 use futures::StreamExt;
 
 use super::*;
-use crate::db::index::queries::{
-    rbac::{get_chain_root::*, get_registrations::*, get_role0_chain_root::*},
-    registrations::{
-        get_from_stake_addr::*, get_from_stake_hash::*, get_from_vote_key::*, get_invalid::*,
+use crate::{
+    db::index::queries::{
+        rbac::{get_chain_root::*, get_registrations::*, get_role0_chain_root::*},
+        registrations::{
+            get_from_stake_addr::*, get_from_stake_hash::*, get_from_vote_key::*, get_invalid::*,
+        },
+        staked_ada::{
+            get_assets_by_stake_address::*, get_txi_by_txn_hash::*, get_txo_by_stake_address::*,
+            update_txo_spent::*,
+        },
+        sync_status::update::*,
     },
-    staked_ada::{
-        get_assets_by_stake_address::*, get_txi_by_txn_hash::*, get_txo_by_stake_address::*,
-        update_txo_spent::*,
-    },
-    sync_status::update::*,
+    service::common::types::cardano::slot_no::SlotNo,
 };
 
 #[ignore = "An integration test which requires a running Scylla node instance, disabled from `testunit` CI run"]
@@ -58,7 +61,7 @@ async fn test_get_invalid_registration_w_stake_addr() {
 
     let mut row_stream = GetInvalidRegistrationQuery::execute(
         &session,
-        GetInvalidRegistrationParams::new(vec![], num_bigint::BigInt::from(i64::MAX)),
+        GetInvalidRegistrationParams::new(vec![], SlotNo::from(u64::MAX)),
     )
     .await
     .unwrap();

catalyst-gateway/bin/src/main.rs

@@ -7,6 +7,7 @@ mod cli;
 mod db;
 mod jinja;
 mod logger;
+mod metrics;
 mod service;
 mod settings;
 mod utils;

catalyst-gateway/bin/src/metrics/chain_follower.rs

@@ -0,0 +1 @@
+//! Metrics related to Chain Follower analytics.

catalyst-gateway/bin/src/metrics/chain_indexer.rs

@@ -0,0 +1 @@
+//! Metrics related to Chain Indexer analytics.

catalyst-gateway/bin/src/metrics/endpoint.rs

@@ -0,0 +1,60 @@
+//! Metrics related to endpoint analytics.
+
+use std::sync::LazyLock;
+
+use prometheus::{register_histogram_vec, register_int_counter_vec, HistogramVec, IntCounterVec};
+
+/// Labels for the metrics
+const METRIC_LABELS: [&str; 3] = ["endpoint", "method", "status_code"];
+/// Labels for the client metrics
+const CLIENT_METRIC_LABELS: [&str; 2] = ["client", "status_code"];
+
+// Prometheus Metrics maintained by the service
+
+/// HTTP Request duration histogram.
+pub(crate) static HTTP_REQ_DURATION_MS: LazyLock<HistogramVec> = LazyLock::new(|| {
+    register_histogram_vec!(
+        "http_request_duration_ms",
+        "Duration of HTTP requests in milliseconds",
+        &METRIC_LABELS
+    )
+    .unwrap()
+});
+
+/// HTTP Request CPU Time histogram.
+pub(crate) static HTTP_REQ_CPU_TIME_MS: LazyLock<HistogramVec> = LazyLock::new(|| {
+    register_histogram_vec!(
+        "http_request_cpu_time_ms",
+        "CPU Time of HTTP requests in milliseconds",
+        &METRIC_LABELS
+    )
+    .unwrap()
+});
+
+// No Tacho implemented to enable this.
+// static ref HTTP_REQUEST_RATE: GaugeVec = register_gauge_vec!(
+// "http_request_rate",
+// "Rate of HTTP requests per second",
+// &METRIC_LABELS
+// )
+// .unwrap();
+
+/// HTTP Request count histogram.
+pub(crate) static HTTP_REQUEST_COUNT: LazyLock<IntCounterVec> = LazyLock::new(|| {
+    register_int_counter_vec!(
+        "http_request_count",
+        "Number of HTTP requests",
+        &METRIC_LABELS
+    )
+    .unwrap()
+});
+
+/// Client Request Count histogram.
+pub(crate) static CLIENT_REQUEST_COUNT: LazyLock<IntCounterVec> = LazyLock::new(|| {
+    register_int_counter_vec!(
+        "client_request_count",
+        "Number of HTTP requests per client",
+        &CLIENT_METRIC_LABELS
+    )
+    .unwrap()
+});

catalyst-gateway/bin/src/metrics/memory.rs

@@ -0,0 +1 @@
+//! Metrics related to memory analytics.

catalyst-gateway/bin/src/metrics/mod.rs

@@ -0,0 +1,18 @@
+//! This module contains submodules related to metrics report and analytics.
+
+use prometheus::{default_registry, Registry};
+
+pub(crate) mod chain_follower;
+pub(crate) mod chain_indexer;
+pub(crate) mod endpoint;
+pub(crate) mod memory;
+
+/// Initialize Prometheus metrics.
+///
+/// ## Returns
+///
+/// Returns the default prometheus registry.
+#[must_use]
+pub(crate) fn init_prometheus() -> Registry {
+    default_registry().clone()
+}