feat: Retrieve gas payments by sequence for scraping Sealevel (#4942)

### Description - Retrieve gas payment by origin, interchain_gas_paymaster and sequence - Add Sealevel to Scraper in E2E Ethereum and Sealevel tests ### Related issues - Contributes into #4271 ### Backward compatibility Yes (after backfill of origin, destination, interchain_gas_paymaster) ### Testing E2E Ethereum and Sealevel tests (including scraping Sealevel) --------- Co-authored-by: Danil Nemirovsky <[email protected]>
hyperlane-xyz · Dec 6, 2024 · 4b280cd · 4b280cd
1 parent a96448f
commit 4b280cd
Show file tree

Hide file tree

Showing 13 changed files with 225 additions and 78 deletions.
diff --git a/rust/main/agents/scraper/migration/src/m20230309_000001_create_table_domain.rs b/rust/main/agents/scraper/migration/src/m20230309_000001_create_table_domain.rs
@@ -462,6 +462,22 @@ const DOMAINS: &[RawDomain] = &[
         is_test_net: true,
         is_deprecated: false,
     },
+    RawDomain {
+        name: "sealeveltest1",
+        token: "SOL",
+        domain: 13375,
+        chain_id: 13375,
+        is_test_net: true,
+        is_deprecated: false,
+    },
+    RawDomain {
+        name: "sealeveltest2",
+        token: "SOL",
+        domain: 13376,
+        chain_id: 13376,
+        is_test_net: true,
+        is_deprecated: false,
+    },
 ];
 
 #[derive(DeriveMigrationName)]

diff --git a/rust/main/agents/scraper/migration/src/m20230309_000004_create_table_gas_payment.rs b/rust/main/agents/scraper/migration/src/m20230309_000004_create_table_gas_payment.rs
@@ -1,5 +1,3 @@
-use std::borrow::BorrowMut as _;
-
 use sea_orm::ConnectionTrait;
 use sea_orm_migration::prelude::*;
 
@@ -41,11 +39,15 @@ impl MigrationTrait for Migration {
                             .big_unsigned()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(GasPayment::Origin).unsigned())
-                    .col(ColumnDef::new(GasPayment::Destination).unsigned())
+                    .col(ColumnDef::new(GasPayment::Origin).unsigned().not_null())
+                    .col(
+                        ColumnDef::new(GasPayment::Destination)
+                            .unsigned()
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new_with_type(GasPayment::InterchainGasPaymaster, Address)
-                            .borrow_mut(),
+                            .not_null(),
                     )
                     .col(ColumnDef::new(GasPayment::Sequence).big_integer())
                     .foreign_key(
@@ -63,11 +65,6 @@ impl MigrationTrait for Migration {
                             .from_col(GasPayment::Origin)
                             .to(Domain::Table, Domain::Id),
                     )
-                    .foreign_key(
-                        ForeignKey::create()
-                            .from_col(GasPayment::Destination)
-                            .to(Domain::Table, Domain::Id),
-                    )
                     .index(
                         Index::create()
                             // don't need domain because TxId includes it
@@ -90,6 +87,19 @@ impl MigrationTrait for Migration {
             )
             .await?;
 
+        manager
+            .create_index(
+                Index::create()
+                    .table(GasPayment::Table)
+                    .name("gas_payment_origin_interchain_gas_paymaster_sequence_idx")
+                    .col(GasPayment::Origin)
+                    .col(GasPayment::InterchainGasPaymaster)
+                    .col(GasPayment::Sequence)
+                    .index_type(IndexType::BTree)
+                    .to_owned(),
+            )
+            .await?;
+
         manager
             .get_connection()
             .execute_unprepared(&format!(

diff --git a/rust/main/agents/scraper/src/agent.rs b/rust/main/agents/scraper/src/agent.rs
@@ -231,7 +231,7 @@ impl Scraper {
         let sync = self
             .as_ref()
             .settings
-            .watermark_contract_sync::<Delivery, _>(
+            .contract_sync::<Delivery, _>(
                 &domain,
                 &metrics.clone(),
                 &contract_sync_metrics.clone(),
@@ -264,11 +264,11 @@ impl Scraper {
         let sync = self
             .as_ref()
             .settings
-            .watermark_contract_sync::<InterchainGasPayment, _>(
+            .contract_sync::<InterchainGasPayment, _>(
                 &domain,
                 &metrics.clone(),
                 &contract_sync_metrics.clone(),
-                Arc::new(store.clone()),
+                Arc::new(store.clone()) as _,
                 true,
             )
             .await

diff --git a/rust/main/agents/scraper/src/conversions.rs b/rust/main/agents/scraper/src/conversions.rs
@@ -8,3 +8,31 @@ pub fn u256_to_decimal(v: U256) -> BigDecimal {
     v.to_little_endian(&mut buf);
     BigDecimal::from(BigInt::from_bytes_le(Sign::Plus, &buf as &[u8]))
 }
+
+pub fn decimal_to_u256(v: BigDecimal) -> U256 {
+    let (i, _) = v.into_bigint_and_exponent();
+    let (_, b) = i.to_bytes_le();
+    U256::from_little_endian(&b)
+}
+
+#[cfg(test)]
+mod tests {
+    use hyperlane_core::U256;
+
+    use crate::conversions::{decimal_to_u256, u256_to_decimal};
+
+    #[test]
+    fn test() {
+        // given
+        let u = U256::from_dec_str(
+            "76418673493495739447102571088210420170780567439841463646292940247514478199569",
+        )
+        .unwrap();
+
+        // when
+        let r = decimal_to_u256(u256_to_decimal(u));
+
+        // then
+        assert_eq!(u, r);
+    }
+}
diff --git a/rust/main/agents/scraper/src/db/generated/gas_payment.rs b/rust/main/agents/scraper/src/db/generated/gas_payment.rs
@@ -21,9 +21,9 @@ pub struct Model {
     pub gas_amount: BigDecimal,
     pub tx_id: i64,
     pub log_index: i64,
-    pub origin: Option<i32>,
-    pub destination: Option<i32>,
-    pub interchain_gas_paymaster: Option<Vec<u8>>,
+    pub origin: i32,
+    pub destination: i32,
+    pub interchain_gas_paymaster: Vec<u8>,
     pub sequence: Option<i64>,
 }
 
@@ -57,7 +57,6 @@ impl PrimaryKeyTrait for PrimaryKey {
 
 #[derive(Copy, Clone, Debug, EnumIter)]
 pub enum Relation {
-    Destination,
     Domain,
     Origin,
     Transaction,
@@ -75,12 +74,10 @@ impl ColumnTrait for Column {
             Self::GasAmount => ColumnType::Decimal(Some((78u32, 0u32))).def(),
             Self::TxId => ColumnType::BigInteger.def(),
             Self::LogIndex => ColumnType::BigInteger.def(),
-            Self::Origin => ColumnType::Integer.def().null(),
-            Self::Destination => ColumnType::Integer.def().null(),
+            Self::Origin => ColumnType::Integer.def(),
+            Self::Destination => ColumnType::Integer.def(),
             Self::InterchainGasPaymaster => {
-                ColumnType::Binary(sea_orm::sea_query::BlobSize::Blob(None))
-                    .def()
-                    .null()
+                ColumnType::Binary(sea_orm::sea_query::BlobSize::Blob(None)).def()
             }
             Self::Sequence => ColumnType::BigInteger.def().null(),
         }
@@ -90,10 +87,6 @@ impl ColumnTrait for Column {
 impl RelationTrait for Relation {
     fn def(&self) -> RelationDef {
         match self {
-            Self::Destination => Entity::belongs_to(super::domain::Entity)
-                .from(Column::Destination)
-                .to(super::domain::Column::Id)
-                .into(),
             Self::Domain => Entity::belongs_to(super::domain::Entity)
                 .from(Column::Domain)
                 .to(super::domain::Column::Id)

diff --git a/rust/main/agents/scraper/src/db/payment.rs b/rust/main/agents/scraper/src/db/payment.rs
@@ -6,7 +6,7 @@ use tracing::{debug, instrument};
 use hyperlane_core::{address_to_bytes, h256_to_bytes, InterchainGasPayment, LogMeta, H256};
 use migration::OnConflict;
 
-use crate::conversions::u256_to_decimal;
+use crate::conversions::{decimal_to_u256, u256_to_decimal};
 use crate::date_time;
 use crate::db::ScraperDb;
 
@@ -22,6 +22,61 @@ pub struct StorablePayment<'a> {
 }
 
 impl ScraperDb {
+    /// Get the payment associated with a sequence.
+    #[instrument(skip(self))]
+    pub async fn retrieve_payment_by_sequence(
+        &self,
+        origin: u32,
+        interchain_gas_paymaster: &H256,
+        sequence: u32,
+    ) -> Result<Option<InterchainGasPayment>> {
+        if let Some(payment) = gas_payment::Entity::find()
+            .filter(gas_payment::Column::Origin.eq(origin))
+            .filter(
+                gas_payment::Column::InterchainGasPaymaster
+                    .eq(address_to_bytes(interchain_gas_paymaster)),
+            )
+            .filter(gas_payment::Column::Sequence.eq(sequence))
+            .one(&self.0)
+            .await?
+        {
+            let payment = InterchainGasPayment {
+                message_id: H256::from_slice(&payment.msg_id),
+                destination: payment.destination as u32,
+                payment: decimal_to_u256(payment.payment),
+                gas_amount: decimal_to_u256(payment.gas_amount),
+            };
+            Ok(Some(payment))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Get the transaction id of the gas payment associated with a sequence.
+    #[instrument(skip(self))]
+    pub async fn retrieve_payment_tx_id(
+        &self,
+        origin: u32,
+        interchain_gas_paymaster: &H256,
+        sequence: u32,
+    ) -> Result<Option<i64>> {
+        if let Some(payment) = gas_payment::Entity::find()
+            .filter(gas_payment::Column::Origin.eq(origin))
+            .filter(
+                gas_payment::Column::InterchainGasPaymaster
+                    .eq(address_to_bytes(interchain_gas_paymaster)),
+            )
+            .filter(gas_payment::Column::Sequence.eq(sequence))
+            .one(&self.0)
+            .await?
+        {
+            let txn_id = payment.tx_id;
+            Ok(Some(txn_id))
+        } else {
+            Ok(None)
+        }
+    }
+
     #[instrument(skip_all)]
     pub async fn store_payments(
         &self,
@@ -44,9 +99,9 @@ impl ScraperDb {
                 gas_amount: Set(u256_to_decimal(storable.payment.gas_amount)),
                 tx_id: Unchanged(storable.txn_id),
                 log_index: Unchanged(storable.meta.log_index.as_u64() as i64),
-                origin: Set(Some(domain as i32)),
-                destination: Set(Some(storable.payment.destination as i32)),
-                interchain_gas_paymaster: Set(Some(interchain_gas_paymaster.clone())),
+                origin: Set(domain as i32),
+                destination: Set(storable.payment.destination as i32),
+                interchain_gas_paymaster: Set(interchain_gas_paymaster.clone()),
                 sequence: Set(storable.sequence),
             })
             .collect_vec();

diff --git a/rust/main/agents/scraper/src/store/payments.rs b/rust/main/agents/scraper/src/store/payments.rs
@@ -5,7 +5,10 @@ use eyre::Result;
 use itertools::Itertools;
 use tracing::debug;
 
-use hyperlane_core::{HyperlaneLogStore, Indexed, InterchainGasPayment, LogMeta, H512};
+use hyperlane_core::{
+    unwrap_or_none_result, HyperlaneLogStore, HyperlaneSequenceAwareIndexerStoreReader, Indexed,
+    InterchainGasPayment, LogMeta, H512,
+};
 
 use crate::db::StorablePayment;
 use crate::store::storage::HyperlaneDbStore;
@@ -65,3 +68,34 @@ impl HyperlaneLogStore<InterchainGasPayment> for HyperlaneDbStore {
         Ok(stored as u32)
     }
 }
+
+#[async_trait]
+impl HyperlaneSequenceAwareIndexerStoreReader<InterchainGasPayment> for HyperlaneDbStore {
+    /// Gets a gas payment by sequence
+    async fn retrieve_by_sequence(&self, sequence: u32) -> Result<Option<InterchainGasPayment>> {
+        let message = self
+            .db
+            .retrieve_payment_by_sequence(
+                self.domain.id(),
+                &self.interchain_gas_paymaster_address,
+                sequence,
+            )
+            .await?;
+        Ok(message)
+    }
+
+    /// Gets the block number at which the log occurred.
+    async fn retrieve_log_block_number_by_sequence(&self, sequence: u32) -> Result<Option<u64>> {
+        let tx_id = unwrap_or_none_result!(
+            self.db
+                .retrieve_payment_tx_id(
+                    self.domain.id(),
+                    &self.interchain_gas_paymaster_address,
+                    sequence,
+                )
+                .await?
+        );
+        let block_id = unwrap_or_none_result!(self.db.retrieve_block_id(tx_id).await?);
+        Ok(self.db.retrieve_block_number(block_id).await?)
+    }
+}
diff --git a/rust/main/config/test_sealevel_config.json b/rust/main/config/test_sealevel_config.json
@@ -15,7 +15,7 @@
       },
       "rpcUrls": [
         {
-          "http": "http://localhost:8899"
+          "http": "http://127.0.0.1:8899"
         }
       ],
       "index": {
@@ -38,7 +38,7 @@
       },
       "rpcUrls": [
         {
-          "http": "http://localhost:8899"
+          "http": "http://127.0.0.1:8899"
         }
       ],
       "index": {