Merge branch 'main' into feat/parse-missing-document-properties

# Conflicts:
#	catalyst_voices/apps/voices/lib/widgets/tiles/document_builder_section_tile.dart
#	catalyst_voices/packages/internal/catalyst_voices_models/lib/src/document/definitions/nested_questions_definition.dart
#	catalyst_voices/packages/internal/catalyst_voices_models/lib/src/document/definitions/single_line_https_url_entry_definition.dart
#	catalyst_voices/packages/internal/catalyst_voices_models/lib/src/document/definitions/yes_no_choice_definition.dart
#	catalyst_voices/packages/internal/catalyst_voices_models/lib/src/document/document_validator.dart
#	catalyst_voices/packages/internal/catalyst_voices_repositories/lib/src/dto/document/schema/document_schema_property_dto.dart

.config/dictionaries/project.dic

@@ -8,6 +8,7 @@ adminer
 afinet
 androidx
 anypolicy
+apirequest
 appbar
 appspot
 Arbritrary
@@ -174,6 +175,7 @@ lovelace
 lovelaces
 LTRB
 Lynxx
+LynxLynxx
 mdlint
 metadatum
 metadatums
@@ -258,7 +260,6 @@ rustflags
 rustfmt
 rustls
 rxdart
-ryszard-schossler
 saibatizoku
 Schemathesis
 Scripthash
@@ -319,6 +320,7 @@ Unstaked
 upskilling
 UTXO
 Utxos
+uuidv
 varint
 Vespr
 vite
@@ -351,3 +353,5 @@ xpub
 xpublic
 xvfb
 yoroi
+stake1u94ullc9nj9gawc08990nx8hwgw80l9zpmr8re44kydqy9cdjq6rq
+invalid1u9nlq5nmuzthw3vhgakfpxyq4r0zl2c0p8uqy24gpyjsa6c3df4h6

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ permissions:
 
 jobs:
   ci:
-    uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.5.1
+    uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.5.3
     with:
       forge_version: 0.8.0
 

.github/workflows/flutter-uikit-example-firebase-deploy.yml

@@ -20,13 +20,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Forge
-        uses: input-output-hk/catalyst-forge/actions/install@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/install@ci/v1.5.3
         with:
           version: 0.8.0
       - name: Setup CI
-        uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.5.3
       - name: Build Flutter Web
-        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3
         if: always()
         continue-on-error: true
         with:

.github/workflows/generate-allure-report.yml

@@ -26,40 +26,40 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Forge
-        uses: input-output-hk/catalyst-forge/actions/install@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/install@ci/v1.5.3
         with:
           version: 0.8.0
         if: always()
 
       - name: Setup CI
-        uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.5.3
 
       - name: Get catalyst gateway unit test report
-        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3
         if: always()
         continue-on-error: true
         with:
           command: run
           args: ./catalyst-gateway+build
 
       - name: Get schemathesis test report
-        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3
         if: always()
         continue-on-error: true
         with:
           command: run
           args: ./catalyst-gateway/tests/schemathesis_tests+test-fuzzer-api
 
       - name: Get flutter unit test report
-        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3
         if: always()
         continue-on-error: true
         with:
           command: run
           args: ./catalyst_voices+test-unit
 
       - name: Get python api test report
-        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.1
+        uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3
         if: always()
         continue-on-error: true
         with:

Earthfile

@@ -1,8 +1,8 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.2.27 AS mdlint-ci
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.2.27 AS cspell-ci
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.27 AS postgresql-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.2.31 AS mdlint-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.2.31 AS cspell-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.31 AS postgresql-ci
 
 FROM debian:stable-slim
 

catalyst-gateway/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.28 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.31 AS rust-ci
 
 #cspell: words rustfmt toolsets USERARCH stdcfgs
 

catalyst-gateway/bin/Cargo.toml

@@ -68,8 +68,8 @@ rust_decimal = { version = "1.36.0", features = [
     "serde-with-float",
     "db-tokio-postgres",
 ] }
-poem = { version = "3.1.3", features = ["embed", "prometheus", "compression"] }
-poem-openapi = { version = "5.1.2", features = [
+poem = { version = "3.1.6", features = ["embed", "prometheus", "compression"] }
+poem-openapi = { version = "5.1.5", features = [
     "openapi-explorer",
     "rapidoc",
     "redoc",
@@ -96,6 +96,8 @@ bech32 = "0.11.0"
 const_format = "0.2.33"
 regex = "1.11.1"
 minijinja = "2.5.0"
+bytes = "1.9.0"
+mime = "0.3.17"
 
 [dev-dependencies]
 proptest = "1.5.0"

catalyst-gateway/bin/src/db/index/queries/cql/get_all_stake_addrs.cql

@@ -0,0 +1,3 @@
+SELECT 
+    stake_address,vote_key
+FROM cip36_registration;

catalyst-gateway/bin/src/db/index/queries/mod.rs

@@ -16,6 +16,7 @@ use rbac::{
     get_role0_chain_root::GetRole0ChainRootQuery,
 };
 use registrations::{
+    get_all_stakes_and_vote_keys::GetAllStakesAndVoteKeysQuery,
     get_from_stake_addr::GetRegistrationQuery, get_from_stake_hash::GetStakeAddrQuery,
     get_from_vote_key::GetStakeAddrFromVoteKeyQuery, get_invalid::GetInvalidRegistrationQuery,
 };
@@ -96,6 +97,8 @@ pub(crate) enum PreparedSelectQuery {
     RegistrationsByChainRoot,
     /// Get chain root by role0 key
     ChainRootByRole0Key,
+    /// Get all stake and vote keys for snapshot (`stake_pub_key,vote_key`)
+    GetAllStakesAndVoteKeys,
 }
 
 /// All prepared UPSERT query statements (inserts/updates a single value of data).
@@ -157,6 +160,8 @@ pub(crate) struct PreparedQueries {
     registrations_by_chain_root_query: PreparedStatement,
     /// Get chain root by role0 key
     chain_root_by_role0_key_query: PreparedStatement,
+    /// Get all stake and vote keys (`stake_key,vote_key`) for snapshot
+    get_all_stakes_and_vote_keys_query: PreparedStatement,
 }
 
 /// An individual query response that can fail
@@ -193,7 +198,9 @@ impl PreparedQueries {
         let chain_root_by_stake_address = GetChainRootQuery::prepare(session.clone()).await;
         let registrations_by_chain_root =
             GetRegistrationsByChainRootQuery::prepare(session.clone()).await;
-        let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session).await;
+        let chain_root_by_role0_key = GetRole0ChainRootQuery::prepare(session.clone()).await;
+        let get_all_stakes_and_vote_keys_query =
+            GetAllStakesAndVoteKeysQuery::prepare(session).await;
 
         let (
             txo_insert_queries,
@@ -241,6 +248,7 @@ impl PreparedQueries {
             chain_root_by_stake_address_query: chain_root_by_stake_address?,
             registrations_by_chain_root_query: registrations_by_chain_root?,
             chain_root_by_role0_key_query: chain_root_by_role0_key?,
+            get_all_stakes_and_vote_keys_query: get_all_stakes_and_vote_keys_query?,
         })
     }
 
@@ -331,6 +339,9 @@ impl PreparedQueries {
                 &self.registrations_by_chain_root_query
             },
             PreparedSelectQuery::ChainRootByRole0Key => &self.chain_root_by_role0_key_query,
+            PreparedSelectQuery::GetAllStakesAndVoteKeys => {
+                &self.get_all_stakes_and_vote_keys_query
+            },
         };
 
         session

catalyst-gateway/bin/src/db/index/queries/registrations/get_all_stakes_and_vote_keys.rs

@@ -0,0 +1,61 @@
+//! Get all stake and vote keys (`stake_pub_key,vote_key`)
+//! Result is used to compose various query registrations for snapshot.
+
+use std::sync::Arc;
+
+use scylla::{
+    prepared_statement::PreparedStatement, transport::iterator::TypedRowStream, DeserializeRow,
+    SerializeRow, Session,
+};
+use tracing::error;
+
+use crate::db::index::{
+    queries::{PreparedQueries, PreparedSelectQuery},
+    session::CassandraSession,
+};
+
+/// Get all (`stake_addr,vote` keys)
+/// [(`stake_addr,vote_key`)]
+const GET_ALL_STAKES_AND_VOTE_KEYS: &str = include_str!("../cql/get_all_stake_addrs.cql");
+
+/// Get all stake and vote keys from cip36 registration
+#[derive(SerializeRow)]
+pub(crate) struct GetAllStakesAndVoteKeysParams {}
+
+/// Get stakes and vote keys for snapshot.
+#[derive(DeserializeRow)]
+pub(crate) struct GetAllStakesAndVoteKeysQuery {
+    /// Full Stake Address (not hashed, 32 byte ED25519 Public key).
+    pub stake_address: Vec<u8>,
+    /// Voting Public Key
+    pub vote_key: Vec<u8>,
+}
+
+impl GetAllStakesAndVoteKeysQuery {
+    /// Prepares get all `stake_addr` paired with vote keys [(`stake_addr,vote_key`)]
+    pub(crate) async fn prepare(session: Arc<Session>) -> anyhow::Result<PreparedStatement> {
+        let get_all_stake_and_vote_keys = PreparedQueries::prepare(
+            session,
+            GET_ALL_STAKES_AND_VOTE_KEYS,
+            scylla::statement::Consistency::All,
+            true,
+        )
+        .await;
+
+        get_all_stake_and_vote_keys.inspect_err(
+            |error| error!(error=%error, "Failed to prepare get all (stake addrs, vote_keys)"),
+        )
+    }
+
+    /// Executes get all `stake_addr` paired with vote keys [(`stake_addr,vote_key`)]
+    pub(crate) async fn execute(
+        session: &CassandraSession, params: GetAllStakesAndVoteKeysParams,
+    ) -> anyhow::Result<TypedRowStream<GetAllStakesAndVoteKeysQuery>> {
+        let iter = session
+            .execute_iter(PreparedSelectQuery::GetAllStakesAndVoteKeys, params)
+            .await?
+            .rows_stream::<GetAllStakesAndVoteKeysQuery>()?;
+
+        Ok(iter)
+    }
+}

catalyst-gateway/bin/src/db/index/queries/registrations/get_from_stake_addr.rs

@@ -24,10 +24,10 @@ pub(crate) struct GetRegistrationParams {
     pub stake_address: Vec<u8>,
 }
 
-impl From<&ed25519_dalek::VerifyingKey> for GetRegistrationParams {
-    fn from(value: &ed25519_dalek::VerifyingKey) -> Self {
+impl From<Vec<u8>> for GetRegistrationParams {
+    fn from(value: Vec<u8>) -> Self {
         GetRegistrationParams {
-            stake_address: value.as_bytes().to_vec(),
+            stake_address: value,
         }
     }
 }

catalyst-gateway/bin/src/db/index/queries/registrations/get_invalid.rs

@@ -8,9 +8,12 @@ use scylla::{
 };
 use tracing::error;
 
-use crate::db::index::{
-    queries::{PreparedQueries, PreparedSelectQuery},
-    session::CassandraSession,
+use crate::{
+    db::index::{
+        queries::{PreparedQueries, PreparedSelectQuery},
+        session::CassandraSession,
+    },
+    service::common::types::cardano::slot_no::SlotNo,
 };
 
 /// Get invalid registrations from stake addr query.
@@ -28,12 +31,10 @@ pub(crate) struct GetInvalidRegistrationParams {
 
 impl GetInvalidRegistrationParams {
     /// Create a new instance of [`GetInvalidRegistrationParams`]
-    pub(crate) fn new(
-        stake_address: Vec<u8>, slot_no: num_bigint::BigInt,
-    ) -> GetInvalidRegistrationParams {
+    pub(crate) fn new(stake_address: Vec<u8>, slot_no: SlotNo) -> GetInvalidRegistrationParams {
         Self {
             stake_address,
-            slot_no,
+            slot_no: slot_no.into(),
         }
     }
 }

catalyst-gateway/bin/src/db/index/queries/registrations/mod.rs

@@ -1,4 +1,5 @@
 //! Registration related queries.
+pub(crate) mod get_all_stakes_and_vote_keys;
 pub(crate) mod get_from_stake_addr;
 pub(crate) mod get_from_stake_hash;
 pub(crate) mod get_from_vote_key;

catalyst-gateway/bin/src/db/index/schema/cql/cip36_registration_for_vote_key.cql

@@ -1,5 +1,5 @@
 -- Index of CIP-36 registrations searchable by Stake Address.
--- Full registration data needs to be queried from the man cip36 registration tables.
+-- Full registration data needs to be queried from the main cip36 registration tables.
 -- Includes both Valid and Invalid registrations.
 CREATE TABLE IF NOT EXISTS cip36_registration_for_vote_key (
     -- Primary Key Data

catalyst-gateway/bin/src/db/index/tests/scylla_queries.rs

@@ -4,16 +4,19 @@
 use futures::StreamExt;
 
 use super::*;
-use crate::db::index::queries::{
-    rbac::{get_chain_root::*, get_registrations::*, get_role0_chain_root::*},
-    registrations::{
-        get_from_stake_addr::*, get_from_stake_hash::*, get_from_vote_key::*, get_invalid::*,
+use crate::{
+    db::index::queries::{
+        rbac::{get_chain_root::*, get_registrations::*, get_role0_chain_root::*},
+        registrations::{
+            get_from_stake_addr::*, get_from_stake_hash::*, get_from_vote_key::*, get_invalid::*,
+        },
+        staked_ada::{
+            get_assets_by_stake_address::*, get_txi_by_txn_hash::*, get_txo_by_stake_address::*,
+            update_txo_spent::*,
+        },
+        sync_status::update::*,
     },
-    staked_ada::{
-        get_assets_by_stake_address::*, get_txi_by_txn_hash::*, get_txo_by_stake_address::*,
-        update_txo_spent::*,
-    },
-    sync_status::update::*,
+    service::common::types::cardano::slot_no::SlotNo,
 };
 
 #[ignore = "An integration test which requires a running Scylla node instance, disabled from `testunit` CI run"]
@@ -58,7 +61,7 @@ async fn test_get_invalid_registration_w_stake_addr() {
 
     let mut row_stream = GetInvalidRegistrationQuery::execute(
         &session,
-        GetInvalidRegistrationParams::new(vec![], num_bigint::BigInt::from(i64::MAX)),
+        GetInvalidRegistrationParams::new(vec![], SlotNo::from(u64::MAX)),
     )
     .await
     .unwrap();

catalyst-gateway/bin/src/main.rs

@@ -7,6 +7,7 @@ mod cli;
 mod db;
 mod jinja;
 mod logger;
+mod metrics;
 mod service;
 mod settings;
 mod utils;

catalyst-gateway/bin/src/metrics/chain_follower.rs

@@ -0,0 +1 @@
+//! Metrics related to Chain Follower analytics.

catalyst-gateway/bin/src/metrics/chain_indexer.rs

@@ -0,0 +1 @@
+//! Metrics related to Chain Indexer analytics.