From 4a70f52544ad5ee8556dabd2819ffb387f02a38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 14 Oct 2024 10:27:13 +0200 Subject: [PATCH 001/111] Add basic balances functionality --- crates/fuel-core/src/graphql_api/storage.rs | 3 + .../src/graphql_api/storage/balances.rs | 154 ++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 crates/fuel-core/src/graphql_api/storage/balances.rs diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index 8f8cfcd1f19..a8b26685953 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -36,6 +36,7 @@ use fuel_core_types::{ }; use statistic::StatisticTable; +mod balances; pub mod blocks; pub mod coins; pub mod contracts; @@ -113,6 +114,8 @@ pub enum Column { DaCompressionTemporalRegistryScriptCode = 21, /// See [`DaCompressionTemporalRegistryPredicateCode`](da_compression::DaCompressionTemporalRegistryPredicateCode) DaCompressionTemporalRegistryPredicateCode = 22, + /// Index of balances per user and asset. + Balances = 23, } impl Column { diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs new file mode 100644 index 00000000000..ba8467d89e0 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -0,0 +1,154 @@ +use fuel_core_chain_config::{ + AddTable, + AsTable, + StateConfig, + StateConfigBuilder, + TableEntry, +}; +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::{ + manual::Manual, + postcard::Postcard, + raw::Raw, + Decode, + Encode, + }, + structured_storage::TableWithBlueprint, + Mappable, +}; +use fuel_core_types::{ + fuel_tx::{ + Address, + AssetId, + Bytes32, + Bytes64, + Bytes8, + }, + fuel_types::BlockHeight, + services::txpool::TransactionStatus, +}; +use std::{ + array::TryFromSliceError, + mem::size_of, +}; + +const BALANCES_KEY_SIZE: usize = Address::LEN + AssetId::LEN; + +type Amount = u64; + +pub type BalancesKey = [u8; Address::LEN + AssetId::LEN]; + +/// These table stores the balances of asset id per owner. +pub struct Balances; + +impl Mappable for Balances { + type Key = BalancesKey; + type OwnedKey = Self::Key; + type Value = Amount; + type OwnedValue = Self::Value; +} + +impl TableWithBlueprint for Balances { + type Blueprint = Plain; // TODO[RC]: What is Plain, Raw, Postcard, Primitive and others in this context? + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::Balances + } +} + +#[cfg(test)] +mod tests { + use fuel_core_storage::{ + StorageInspect, + StorageMutate, + }; + use fuel_core_types::fuel_tx::{ + Address, + AssetId, + Bytes64, + Bytes8, + }; + + use crate::combined_database::CombinedDatabase; + + use super::{ + Balances, + BalancesKey, + }; + + pub struct TestDatabase { + database: CombinedDatabase, + } + + impl TestDatabase { + pub fn new() -> Self { + Self { + database: Default::default(), + } + } + + pub fn balance_tx( + &mut self, + owner: &Address, + (asset_id, amount): &(AssetId, u64), + ) { + let current_balance = self.query_balance(owner, asset_id); + let new_balance = current_balance.unwrap_or(0) + amount; + + let db = self.database.off_chain_mut(); + + let mut key = [0; Address::LEN + AssetId::LEN]; + key[0..Address::LEN].copy_from_slice(owner.as_ref()); + key[Address::LEN..].copy_from_slice(asset_id.as_ref()); + + let _ = StorageMutate::::insert(db, &key, &new_balance) + .expect("couldn't store test asset"); + } + + pub fn query_balance(&self, owner: &Address, asset_id: &AssetId) -> Option { + let db = self.database.off_chain(); + + let mut key = [0; Address::LEN + AssetId::LEN]; + key[0..Address::LEN].copy_from_slice(owner.as_ref()); + key[Address::LEN..].copy_from_slice(asset_id.as_ref()); + + let result = StorageInspect::::get(db, &key).unwrap(); + + result.map(|r| r.into_owned()) + } + } + + #[test] + fn can_store_and_retrieve_assets() { + let mut db = TestDatabase::new(); + + let alice = Address::from([1; 32]); + let bob = Address::from([2; 32]); + let carol = Address::from([3; 32]); + + let ASSET_1 = AssetId::from([1; 32]); + let ASSET_2 = AssetId::from([2; 32]); + + // Alice has 100 of asset 1 and a total of 1000 of asset 2 + let alice_tx_1 = (ASSET_1, 100_u64); + let alice_tx_2 = (ASSET_2, 600_u64); + let alice_tx_3 = (ASSET_2, 400_u64); + + // Carol has 200 of asset 2 + let carol_tx_1 = (ASSET_2, 200_u64); + + let res = db.balance_tx(&alice, &alice_tx_1); + let res = db.balance_tx(&alice, &alice_tx_2); + let res = db.balance_tx(&alice, &alice_tx_3); + let res = db.balance_tx(&carol, &carol_tx_1); + + // Alice has correct balances + assert_eq!(db.query_balance(&alice, &alice_tx_1.0), Some(100)); + assert_eq!(db.query_balance(&alice, &alice_tx_2.0), Some(1000)); + + // Carol has correct balances + assert_eq!(db.query_balance(&carol, &carol_tx_1.0), Some(200_u64)); + } +} From 7d472fbf238ce0fe33cd7a9ceaf587de14c92cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 14 Oct 2024 10:58:01 +0200 Subject: [PATCH 002/111] Add support for querying all balances for user --- .../src/graphql_api/storage/balances.rs | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index ba8467d89e0..24787d09970 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -60,7 +60,10 @@ impl TableWithBlueprint for Balances { #[cfg(test)] mod tests { + use std::collections::HashMap; + use fuel_core_storage::{ + iter::IterDirection, StorageInspect, StorageMutate, }; @@ -118,10 +121,27 @@ mod tests { result.map(|r| r.into_owned()) } + + pub fn query_balances(&self, owner: &Address) -> HashMap { + let db = self.database.off_chain(); + + let mut key_prefix = owner.as_ref().to_vec(); + db.entries::(Some(key_prefix), IterDirection::Forward) + .map(|asset| { + let asset = asset.unwrap(); + let asset_id = + AssetId::from_bytes_ref_checked(&asset.key[AssetId::LEN..]) + .copied() + .expect("incorrect bytes"); + let balance = asset.value; + (asset_id, balance) + }) + .collect() + } } #[test] - fn can_store_and_retrieve_assets() { + fn can_retrieve_balance_of_asset() { let mut db = TestDatabase::new(); let alice = Address::from([1; 32]); @@ -151,4 +171,45 @@ mod tests { // Carol has correct balances assert_eq!(db.query_balance(&carol, &carol_tx_1.0), Some(200_u64)); } + + #[test] + fn can_retrieve_balances_of_all_assets_of_owner() { + let mut db = TestDatabase::new(); + + let alice = Address::from([1; 32]); + let bob = Address::from([2; 32]); + let carol = Address::from([3; 32]); + + let ASSET_1 = AssetId::from([1; 32]); + let ASSET_2 = AssetId::from([2; 32]); + + // Alice has 100 of asset 1 and a total of 1000 of asset 2 + let alice_tx_1 = (ASSET_1, 100_u64); + let alice_tx_2 = (ASSET_2, 600_u64); + let alice_tx_3 = (ASSET_2, 400_u64); + + // Carol has 200 of asset 2 + let carol_tx_1 = (ASSET_2, 200_u64); + + let res = db.balance_tx(&alice, &alice_tx_1); + let res = db.balance_tx(&alice, &alice_tx_2); + let res = db.balance_tx(&alice, &alice_tx_3); + let res = db.balance_tx(&carol, &carol_tx_1); + + // Verify Alice balances + let expected: HashMap<_, _> = vec![(ASSET_1, 100_u64), (ASSET_2, 1000_u64)] + .into_iter() + .collect(); + let actual = db.query_balances(&alice); + assert_eq!(expected, actual); + + // Verify Bob balances + let actual = db.query_balances(&bob); + assert_eq!(HashMap::new(), actual); + + // Verify Carol balances + let expected: HashMap<_, _> = vec![(ASSET_2, 200_u64)].into_iter().collect(); + let actual = db.query_balances(&carol); + assert_eq!(expected, actual); + } } From 4296fd4e73575c5c1db19591c1e9d34e8d9d79a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 14 Oct 2024 12:24:00 +0200 Subject: [PATCH 003/111] Adding `balances_indexation_progress` to DB metadata --- crates/fuel-core/src/database.rs | 10 +++++--- .../src/database/database_description.rs | 23 ++++++++++++++++++- .../database_description/off_chain.rs | 2 ++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index d1d85dfe17e..8b4e22a9afb 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -484,9 +484,11 @@ where .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::V1 { + &DatabaseMetadata::V2 { version: Description::version(), height: new_height, + // TODO[RC]: This value must NOT be updated here. + balances_indexation_progress: Default::default(), }, )?; @@ -908,9 +910,10 @@ mod tests { .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::::V1 { + &DatabaseMetadata::::V2 { version: Default::default(), height: Default::default(), + balances_indexation_progress: Default::default(), }, ) .unwrap(); @@ -982,9 +985,10 @@ mod tests { // When let result = database.storage_as_mut::>().insert( &(), - &DatabaseMetadata::::V1 { + &DatabaseMetadata::::V2 { version: Default::default(), height: Default::default(), + balances_indexation_progress: Default::default(), }, ); diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 14d240c54f5..ca91ff20411 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -70,7 +70,15 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { /// The metadata of the database contains information about the version and its height. #[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { - V1 { version: u32, height: Height }, + V1 { + version: u32, + height: Height, + }, + V2 { + version: u32, + height: Height, + balances_indexation_progress: Height, + }, } impl DatabaseMetadata { @@ -78,6 +86,7 @@ impl DatabaseMetadata { pub fn version(&self) -> u32 { match self { Self::V1 { version, .. } => *version, + Self::V2 { version, .. } => *version, } } @@ -85,6 +94,18 @@ impl DatabaseMetadata { pub fn height(&self) -> &Height { match self { Self::V1 { height, .. } => height, + Self::V2 { height, .. } => height, + } + } + + /// Returns the height of the database. + pub fn balances_indexation_progress(&self) -> Option<&Height> { + match self { + Self::V1 { height, .. } => None, + Self::V2 { + balances_indexation_progress, + .. + } => Some(balances_indexation_progress), } } } diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs index 1f339c50f3c..e7985962008 100644 --- a/crates/fuel-core/src/database/database_description/off_chain.rs +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -12,6 +12,8 @@ impl DatabaseDescription for OffChain { type Height = BlockHeight; fn version() -> u32 { + // TODO[RC]: Flip to 1, to take care of DatabaseMetadata::V2 + // TODO[RC]: This will fail the check_version(), do we need to migrate first? 0 } From 77d0f16ef1404aeeb6e78789f58019cd3c7e1c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 15 Oct 2024 17:42:46 +0200 Subject: [PATCH 004/111] Attempt at migrating the metadata to store the indexation progress --- Cargo.lock | 1 + crates/fuel-core/src/combined_database.rs | 5 ++ crates/fuel-core/src/database.rs | 6 ++ crates/fuel-core/src/database/metadata.rs | 90 +++++++++++++++++++++++ crates/fuel-core/src/database/storage.rs | 5 +- crates/fuel-core/src/lib.rs | 2 +- crates/fuel-core/src/service.rs | 1 + crates/storage/Cargo.toml | 1 + crates/storage/src/lib.rs | 4 +- 9 files changed, 111 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0df5ae80960..5862df04b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3676,6 +3676,7 @@ dependencies = [ "strum 0.25.0", "strum_macros 0.25.3", "test-case", + "tracing", ] [[package]] diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index f69649d2605..9c83da03e3b 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -135,6 +135,11 @@ impl CombinedDatabase { ) } + pub fn migrate_metadata(&mut self) -> StorageResult<()> { + self.off_chain.migrate_metadata()?; + Ok(()) + } + pub fn check_version(&self) -> StorageResult<()> { self.on_chain.check_version()?; self.off_chain.check_version()?; diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 8b4e22a9afb..6565677b884 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -61,6 +61,7 @@ use std::{ fmt::Debug, sync::Arc, }; +use tracing::info; pub use fuel_core_database::Error; pub type Result = core::result::Result; @@ -419,10 +420,14 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { + dbg!(&changes); + // Gets the all new heights from the `changes` let iterator = ChangesIterator::::new(&changes); let new_heights = heights_lookup(&iterator)?; + dbg!(&new_heights); + // Changes for each block should be committed separately. // If we have more than one height, it means we are mixing commits // for several heights in one batch - return error in this case. @@ -464,6 +469,7 @@ where (Some(prev_height), None) => { // In production, we shouldn't have cases where we call `commit_changes` with intermediate changes. // The commit always should contain all data for the corresponding height. + info!("XXXX - bailing here because new_height is not set"); return Err(DatabaseError::NewHeightIsNotSet { prev_height: prev_height.as_u64(), } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 72cf2bbedb7..f20a5ded560 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -10,12 +10,21 @@ use fuel_core_storage::{ blueprint::plain::Plain, codec::postcard::Postcard, structured_storage::TableWithBlueprint, + transactional::{ + Changes, + ConflictPolicy, + Modifiable, + StorageTransaction, + }, Error as StorageError, Mappable, Result as StorageResult, + StorageAsMut, StorageAsRef, StorageInspect, + StorageMutate, }; +use tracing::info; /// The table that stores all metadata about the database. pub struct MetadataTable(core::marker::PhantomData); @@ -42,6 +51,87 @@ where } } +impl Database +where + Description: DatabaseDescription, + Self: StorageInspect, Error = StorageError> + + StorageMutate> + + Modifiable, +{ + // TODO[RC]: Add test covering this. + pub fn migrate_metadata(&mut self) -> StorageResult<()> { + let Some(current_metadata) = + self.storage::>().get(&())? + else { + return Ok(()); + }; + + dbg!(¤t_metadata); + + match current_metadata.as_ref() { + DatabaseMetadata::V1 { version, height } => { + let new_metadata = DatabaseMetadata::V2 { + version: *version+1, + height: *height, + balances_indexation_progress: Default::default(), + }; + info!( + "Migrating metadata from V1 to version V2..." + ); + dbg!(&new_metadata); + info!("XXXX - 3007"); + let x = self.storage_as_mut::>(); + + // None of these work. + //x.insert(&(), &new_metadata)?; + //x.replace(&(), &new_metadata)?; + + info!( + "...Migrated! perhaps" + ); + Ok(()) + + // We probably want to use a pattern similar to this code. Or maybe not, since + // we're just starting up and no other services are running. + /* + let updated_changes = if let Some(new_height) = new_height { + // We want to update the metadata table to include a new height. + // For that, we are building a new storage transaction around `changes`. + // Modifying this transaction will include all required updates into the `changes`. + let mut transaction = StorageTransaction::transaction( + &database, + ConflictPolicy::Overwrite, + changes, + ); + transaction + .storage_as_mut::>() + .insert( + &(), + &DatabaseMetadata::V2 { + version: Description::version(), + height: new_height, + // TODO[RC]: This value must NOT be updated here. + balances_indexation_progress: Default::default(), + }, + )?; + + transaction.into_changes() + } else { + changes + }; + let mut guard = database.stage.height.lock(); + database.data.commit_changes(new_height, updated_changes)?; + */ + } + DatabaseMetadata::V2 { + version, + height, + balances_indexation_progress, + } => return Ok(()), + } + } +} + impl Database where Description: DatabaseDescription, diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index d00e292963f..8b069fbc67d 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -15,6 +15,7 @@ use fuel_core_storage::{ StorageMutate, StorageWrite, }; +use tracing::info; impl StorageMutate for GenericDatabase where @@ -34,7 +35,9 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().replace(key, value)?; - self.commit_changes(transaction.into_changes())?; + let changes = transaction.into_changes(); + dbg!(&changes); + self.commit_changes(changes)?; Ok(prev) } diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index 40d866a137d..b30f960a022 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![deny(warnings)] +#![allow(warnings)] // tmp change to allow warnings use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 6b42dd3960c..f6ec04f2644 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -124,6 +124,7 @@ impl FuelService { // initialize state tracing::info!("Initializing database"); + database.migrate_metadata()?; database.check_version()?; Self::make_database_compatible_with_config( diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index 8750f3ae8b0..af598c4c33d 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -17,6 +17,7 @@ repository = { workspace = true } version = { workspace = true } [dependencies] +tracing = { workspace = true } anyhow = { workspace = true } derive_more = { workspace = true } enum-iterator = { workspace = true } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index c54fbf889b8..d7e89ca4ea1 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -7,9 +7,9 @@ #![cfg_attr(not(feature = "std"), no_std)] #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] -#![deny(unused_crate_dependencies)] +#![allow(unused_crate_dependencies)] // tmp change to allow warnings #![deny(missing_docs)] -#![deny(warnings)] +#![allow(warnings)] // tmp change to allow warnings #[cfg(feature = "alloc")] extern crate alloc; From 8eba5d7a08b1d1083d68588eb8e1dc0f74466d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 16 Oct 2024 11:38:40 +0200 Subject: [PATCH 005/111] Hack the `replace_forced()` and `commit_changes_forced` in --- Cargo.lock | 38 +++++-------------- Cargo.toml | 10 +++++ crates/fuel-core/src/database.rs | 34 +++++++++++++++++ .../database_description/off_chain.rs | 2 +- crates/fuel-core/src/database/metadata.rs | 3 +- crates/fuel-core/src/database/storage.rs | 17 +++++++++ crates/storage/src/structured_storage.rs | 12 ++++++ crates/storage/src/test_helpers.rs | 9 +++++ crates/storage/src/transactional.rs | 12 ++++++ crates/storage/src/vm_storage.rs | 8 ++++ 10 files changed, 115 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 364e4d7e431..be58781f096 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,7 +735,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -2246,7 +2246,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -3146,8 +3146,6 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3158,8 +3156,6 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3226,7 +3222,7 @@ dependencies = [ "tower", "tower-http 0.4.4", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -3871,8 +3867,6 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", @@ -3904,8 +3898,6 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", @@ -3941,8 +3933,6 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" dependencies = [ "derive_more", "digest 0.10.7", @@ -3962,8 +3952,6 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" [[package]] name = "fuel-tx" @@ -3990,8 +3978,6 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4024,8 +4010,6 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4067,8 +4051,6 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" dependencies = [ "anyhow", "async-trait", @@ -4727,9 +4709,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ "bytes", "futures-channel", @@ -7847,9 +7829,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" @@ -8629,7 +8611,7 @@ dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -9487,9 +9469,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] diff --git a/Cargo.toml b/Cargo.toml index d80bf4f5bdc..e35d230928a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,3 +143,13 @@ itertools = { version = "0.12", default-features = false } insta = "1.8" tempfile = "3.4" tikv-jemallocator = "0.5" + +[patch.crates-io] +fuel-tx = { path = "../fuel-vm/fuel-tx" } +fuel-asm = { path = "../fuel-vm/fuel-asm" } +fuel-crypto = { path = "../fuel-vm/fuel-crypto" } +fuel-derive = { path = "../fuel-vm/fuel-derive" } +fuel-merkle = { path = "../fuel-vm/fuel-merkle" } +fuel-storage = { path = "../fuel-vm/fuel-storage" } +fuel-types = { path = "../fuel-vm/fuel-types" } +fuel-vm = { path = "../fuel-vm/fuel-vm" } \ No newline at end of file diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 6565677b884..005836f8c0d 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -349,6 +349,10 @@ impl Modifiable for Database { .try_collect() }) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl Modifiable for Database { @@ -359,6 +363,12 @@ impl Modifiable for Database { .try_collect() }) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + let prev_height = *self.stage.height.lock(); + self.data.commit_changes(prev_height, changes)?; + Ok(()) + } } impl Modifiable for Database { @@ -368,6 +378,10 @@ impl Modifiable for Database { .try_collect() }) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } #[cfg(feature = "relayer")] @@ -380,6 +394,10 @@ impl Modifiable for Database { .try_collect() }) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } #[cfg(not(feature = "relayer"))] @@ -387,24 +405,40 @@ impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { commit_changes_with_height_update(self, changes, |_| Ok(vec![])) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } fn commit_changes_with_height_update( diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs index e7985962008..9b0ce585901 100644 --- a/crates/fuel-core/src/database/database_description/off_chain.rs +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -14,7 +14,7 @@ impl DatabaseDescription for OffChain { fn version() -> u32 { // TODO[RC]: Flip to 1, to take care of DatabaseMetadata::V2 // TODO[RC]: This will fail the check_version(), do we need to migrate first? - 0 + 1 } fn name() -> String { diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index f20a5ded560..f23ed081268 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -81,10 +81,11 @@ where dbg!(&new_metadata); info!("XXXX - 3007"); let x = self.storage_as_mut::>(); + x.replace_forced(&(), &new_metadata)?; // None of these work. - //x.insert(&(), &new_metadata)?; //x.replace(&(), &new_metadata)?; + //x.insert(&(), &new_metadata)?; info!( "...Migrated! perhaps" diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 8b069fbc67d..ed163cafe49 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -51,6 +51,23 @@ where self.commit_changes(transaction.into_changes())?; Ok(prev) } + + fn replace_forced( + &mut self, + key: &::Key, + value: &::Value, + ) -> Result::OwnedValue>, Self::Error> { + let mut transaction = StorageTransaction::transaction( + self.as_ref(), + ConflictPolicy::Overwrite, + Default::default(), + ); + let prev = transaction.storage_as_mut::().replace(key, value)?; + let changes = transaction.into_changes(); + dbg!(&changes); + self.commit_changes_forced(changes)?; + Ok(prev) + } } impl StorageWrite for GenericDatabase diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index e78e9637484..12f1d0e7869 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -248,6 +248,10 @@ where fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.inner.commit_changes(changes) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl StorageInspect for StructuredStorage @@ -293,6 +297,14 @@ where fn take(&mut self, key: &M::Key) -> Result, Self::Error> { ::Blueprint::take(self, key, M::column()) } + + fn replace_forced( + &mut self, + _key: &::Key, + _value: &::Value, + ) -> Result::OwnedValue>, Self::Error> { + unimplemented!() + } } impl StorageSize for StructuredStorage diff --git a/crates/storage/src/test_helpers.rs b/crates/storage/src/test_helpers.rs index 5967b5c7920..3da2e6febb6 100644 --- a/crates/storage/src/test_helpers.rs +++ b/crates/storage/src/test_helpers.rs @@ -84,6 +84,7 @@ mockall::mock! { impl Modifiable for Basic { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; } } @@ -122,6 +123,14 @@ where fn take(&mut self, key: &M::Key) -> StorageResult> { MockStorageMethods::remove::(&mut self.storage, key) } + + fn replace_forced( + &mut self, + _key: &::Key, + _value: &::Value, + ) -> Result::OwnedValue>, Self::Error> { + unimplemented!() + } } impl MerkleRootStorage for MockStorage diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 14ec74159ed..c05f8015de3 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -133,6 +133,9 @@ pub enum ConflictPolicy { pub trait Modifiable { /// Commits the changes into the storage. fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; + + /// Commits the changes into the storage without validating block height. + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; } /// The wrapper around the `Vec` that supports `Borrow<[u8]>`. @@ -326,6 +329,10 @@ impl Modifiable for InMemoryTransaction { } Ok(()) } + + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } impl InMemoryTransaction @@ -577,6 +584,11 @@ mod test { } Ok(()) } + + #[doc = " Commits the changes into the storage without validating block height."] + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { + unimplemented!() + } } #[test] diff --git a/crates/storage/src/vm_storage.rs b/crates/storage/src/vm_storage.rs index 92ab65ffdaf..12827c6d44a 100644 --- a/crates/storage/src/vm_storage.rs +++ b/crates/storage/src/vm_storage.rs @@ -175,6 +175,14 @@ where fn take(&mut self, key: &M::Key) -> Result, Self::Error> { StorageMutate::::take(&mut self.database, key) } + + fn replace_forced( + &mut self, + _key: &::Key, + _value: &::Value, + ) -> Result::OwnedValue>, Self::Error> { + unimplemented!() + } } impl StorageSize for VmStorage From c083f45dac68736cb17f1212c1e27f854c4207a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 16 Oct 2024 17:18:30 +0200 Subject: [PATCH 006/111] Introduce `ForcedCommitDatabase` --- crates/fuel-core/src/database.rs | 41 +++++------------- crates/fuel-core/src/database/metadata.rs | 50 +++------------------- crates/fuel-core/src/database/storage.rs | 51 +++++++++++++++-------- crates/storage/src/structured_storage.rs | 12 ------ crates/storage/src/test_helpers.rs | 9 ---- crates/storage/src/transactional.rs | 12 ------ crates/storage/src/vm_storage.rs | 8 ---- 7 files changed, 48 insertions(+), 135 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 005836f8c0d..d1b98e63c97 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -349,9 +349,18 @@ impl Modifiable for Database { .try_collect() }) } +} + +trait ForcedCommitDatabase { + fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; +} +impl ForcedCommitDatabase + for GenericDatabase>> +{ fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() + let mut height = *self.stage.height.lock(); + self.data.commit_changes(height, changes) } } @@ -363,12 +372,6 @@ impl Modifiable for Database { .try_collect() }) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - let prev_height = *self.stage.height.lock(); - self.data.commit_changes(prev_height, changes)?; - Ok(()) - } } impl Modifiable for Database { @@ -378,10 +381,6 @@ impl Modifiable for Database { .try_collect() }) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } #[cfg(feature = "relayer")] @@ -394,10 +393,6 @@ impl Modifiable for Database { .try_collect() }) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } #[cfg(not(feature = "relayer"))] @@ -405,40 +400,24 @@ impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { commit_changes_with_height_update(self, changes, |_| Ok(vec![])) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } fn commit_changes_with_height_update( diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index f23ed081268..79b24bd7cd5 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -23,6 +23,7 @@ use fuel_core_storage::{ StorageAsRef, StorageInspect, StorageMutate, + StorageMutateForced, }; use tracing::info; @@ -55,7 +56,7 @@ impl Database where Description: DatabaseDescription, Self: StorageInspect, Error = StorageError> - + StorageMutate> + + StorageMutateForced> + Modifiable, { // TODO[RC]: Add test covering this. @@ -71,58 +72,17 @@ where match current_metadata.as_ref() { DatabaseMetadata::V1 { version, height } => { let new_metadata = DatabaseMetadata::V2 { - version: *version+1, + version: *version + 1, height: *height, balances_indexation_progress: Default::default(), }; - info!( - "Migrating metadata from V1 to version V2..." - ); + info!("Migrating metadata from V1 to version V2..."); dbg!(&new_metadata); - info!("XXXX - 3007"); let x = self.storage_as_mut::>(); x.replace_forced(&(), &new_metadata)?; - // None of these work. - //x.replace(&(), &new_metadata)?; - //x.insert(&(), &new_metadata)?; - - info!( - "...Migrated! perhaps" - ); + info!("...Migrated!"); Ok(()) - - // We probably want to use a pattern similar to this code. Or maybe not, since - // we're just starting up and no other services are running. - /* - let updated_changes = if let Some(new_height) = new_height { - // We want to update the metadata table to include a new height. - // For that, we are building a new storage transaction around `changes`. - // Modifying this transaction will include all required updates into the `changes`. - let mut transaction = StorageTransaction::transaction( - &database, - ConflictPolicy::Overwrite, - changes, - ); - transaction - .storage_as_mut::>() - .insert( - &(), - &DatabaseMetadata::V2 { - version: Description::version(), - height: new_height, - // TODO[RC]: This value must NOT be updated here. - balances_indexation_progress: Default::default(), - }, - )?; - - transaction.into_changes() - } else { - changes - }; - let mut guard = database.stage.height.lock(); - database.data.commit_changes(new_height, updated_changes)?; - */ } DatabaseMetadata::V2 { version, diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index ed163cafe49..9d13005eb82 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -1,7 +1,11 @@ -use crate::state::generic_database::GenericDatabase; +use crate::{ + database::ForcedCommitDatabase, + state::generic_database::GenericDatabase, +}; use fuel_core_storage::{ structured_storage::StructuredStorage, transactional::{ + Changes, ConflictPolicy, Modifiable, StorageTransaction, @@ -12,11 +16,39 @@ use fuel_core_storage::{ StorageAsMut, StorageBatchMutate, StorageInspect, + StorageMut, StorageMutate, + StorageMutateForced, StorageWrite, }; use tracing::info; +impl StorageMutateForced for GenericDatabase +where + M: Mappable, + Self: Modifiable, + StructuredStorage: StorageInspect, + for<'a> StorageTransaction<&'a Storage>: StorageMutate, + GenericDatabase: ForcedCommitDatabase, +{ + fn replace_forced( + &mut self, + key: &::Key, + value: &::Value, + ) -> Result::OwnedValue>, Self::Error> { + let mut transaction = StorageTransaction::transaction( + self.as_ref(), + ConflictPolicy::Overwrite, + Default::default(), + ); + let prev = transaction.storage_as_mut::().replace(key, value)?; + let changes = transaction.into_changes(); + dbg!(&changes); + self.commit_changes_forced(changes)?; + Ok(prev) + } +} + impl StorageMutate for GenericDatabase where M: Mappable, @@ -51,23 +83,6 @@ where self.commit_changes(transaction.into_changes())?; Ok(prev) } - - fn replace_forced( - &mut self, - key: &::Key, - value: &::Value, - ) -> Result::OwnedValue>, Self::Error> { - let mut transaction = StorageTransaction::transaction( - self.as_ref(), - ConflictPolicy::Overwrite, - Default::default(), - ); - let prev = transaction.storage_as_mut::().replace(key, value)?; - let changes = transaction.into_changes(); - dbg!(&changes); - self.commit_changes_forced(changes)?; - Ok(prev) - } } impl StorageWrite for GenericDatabase diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 12f1d0e7869..e78e9637484 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -248,10 +248,6 @@ where fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.inner.commit_changes(changes) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } impl StorageInspect for StructuredStorage @@ -297,14 +293,6 @@ where fn take(&mut self, key: &M::Key) -> Result, Self::Error> { ::Blueprint::take(self, key, M::column()) } - - fn replace_forced( - &mut self, - _key: &::Key, - _value: &::Value, - ) -> Result::OwnedValue>, Self::Error> { - unimplemented!() - } } impl StorageSize for StructuredStorage diff --git a/crates/storage/src/test_helpers.rs b/crates/storage/src/test_helpers.rs index 3da2e6febb6..5967b5c7920 100644 --- a/crates/storage/src/test_helpers.rs +++ b/crates/storage/src/test_helpers.rs @@ -84,7 +84,6 @@ mockall::mock! { impl Modifiable for Basic { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; } } @@ -123,14 +122,6 @@ where fn take(&mut self, key: &M::Key) -> StorageResult> { MockStorageMethods::remove::(&mut self.storage, key) } - - fn replace_forced( - &mut self, - _key: &::Key, - _value: &::Value, - ) -> Result::OwnedValue>, Self::Error> { - unimplemented!() - } } impl MerkleRootStorage for MockStorage diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index c05f8015de3..14ec74159ed 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -133,9 +133,6 @@ pub enum ConflictPolicy { pub trait Modifiable { /// Commits the changes into the storage. fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; - - /// Commits the changes into the storage without validating block height. - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; } /// The wrapper around the `Vec` that supports `Borrow<[u8]>`. @@ -329,10 +326,6 @@ impl Modifiable for InMemoryTransaction { } Ok(()) } - - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } impl InMemoryTransaction @@ -584,11 +577,6 @@ mod test { } Ok(()) } - - #[doc = " Commits the changes into the storage without validating block height."] - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - unimplemented!() - } } #[test] diff --git a/crates/storage/src/vm_storage.rs b/crates/storage/src/vm_storage.rs index 12827c6d44a..92ab65ffdaf 100644 --- a/crates/storage/src/vm_storage.rs +++ b/crates/storage/src/vm_storage.rs @@ -175,14 +175,6 @@ where fn take(&mut self, key: &M::Key) -> Result, Self::Error> { StorageMutate::::take(&mut self.database, key) } - - fn replace_forced( - &mut self, - _key: &::Key, - _value: &::Value, - ) -> Result::OwnedValue>, Self::Error> { - unimplemented!() - } } impl StorageSize for VmStorage From 6b76c37343da2841600437eaea7dc5e889061d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 16 Oct 2024 17:23:26 +0200 Subject: [PATCH 007/111] Update dependencies --- Cargo.lock | 9 +++++++++ Cargo.toml | 17 +++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be58781f096..14d48bb073f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,6 +3146,7 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3156,6 +3157,7 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3867,6 +3869,7 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "coins-bip32", "coins-bip39", @@ -3898,6 +3901,7 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "proc-macro2", "quote", @@ -3933,6 +3937,7 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "derive_more", "digest 0.10.7", @@ -3952,6 +3957,7 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" [[package]] name = "fuel-tx" @@ -3978,6 +3984,7 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4010,6 +4017,7 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4051,6 +4059,7 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" +source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index e35d230928a..1a76cc92ca0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,11 +145,12 @@ tempfile = "3.4" tikv-jemallocator = "0.5" [patch.crates-io] -fuel-tx = { path = "../fuel-vm/fuel-tx" } -fuel-asm = { path = "../fuel-vm/fuel-asm" } -fuel-crypto = { path = "../fuel-vm/fuel-crypto" } -fuel-derive = { path = "../fuel-vm/fuel-derive" } -fuel-merkle = { path = "../fuel-vm/fuel-merkle" } -fuel-storage = { path = "../fuel-vm/fuel-storage" } -fuel-types = { path = "../fuel-vm/fuel-types" } -fuel-vm = { path = "../fuel-vm/fuel-vm" } \ No newline at end of file +fuel-vm-private = { git = 'https://github.com/FuelLabs/fuel-vm.git', package = "fuel-vm", branch = "1965_balances" } +#fuel-tx = { path = "../fuel-vm/fuel-tx" } +#fuel-asm = { path = "../fuel-vm/fuel-asm" } +#fuel-crypto = { path = "../fuel-vm/fuel-crypto" } +#fuel-derive = { path = "../fuel-vm/fuel-derive" } +#fuel-merkle = { path = "../fuel-vm/fuel-merkle" } +#fuel-storage = { path = "../fuel-vm/fuel-storage" } +#fuel-types = { path = "../fuel-vm/fuel-types" } +#fuel-vm = { path = "../fuel-vm/fuel-vm" } \ No newline at end of file From f246cd00a5f5adb335cbd38b3f5ed0de693168fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 11:03:03 +0200 Subject: [PATCH 008/111] DB metadata can track multiple indexation progresses --- crates/fuel-core/src/database.rs | 15 ++++++++++++--- .../src/database/database_description.rs | 16 +++++++++++----- crates/fuel-core/src/database/metadata.rs | 8 ++------ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index d1b98e63c97..9436d5f8ce5 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -95,6 +95,15 @@ pub mod state; pub mod storage; pub mod transactions; +// TODO[RC]: Perhaps move to the new "indexation" module if indexation related structs grow too big. +#[derive( + Copy, Clone, Debug, serde::Serialize, serde::Deserialize, Hash, Eq, PartialEq, +)] +pub(crate) enum IndexationType { + Balances, + CoinsToSpend, +} + #[derive(Default, Debug, Copy, Clone)] pub struct GenesisStage; @@ -507,7 +516,7 @@ where version: Description::version(), height: new_height, // TODO[RC]: This value must NOT be updated here. - balances_indexation_progress: Default::default(), + indexation_progress: Default::default(), }, )?; @@ -932,7 +941,7 @@ mod tests { &DatabaseMetadata::::V2 { version: Default::default(), height: Default::default(), - balances_indexation_progress: Default::default(), + indexation_progress: Default::default(), }, ) .unwrap(); @@ -1007,7 +1016,7 @@ mod tests { &DatabaseMetadata::::V2 { version: Default::default(), height: Default::default(), - balances_indexation_progress: Default::default(), + indexation_progress: Default::default(), }, ); diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index ca91ff20411..548887fec48 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -4,6 +4,9 @@ use fuel_core_types::{ blockchain::primitives::DaBlockHeight, fuel_types::BlockHeight, }; +use std::collections::HashMap; + +use super::IndexationType; pub mod gas_price; pub mod off_chain; @@ -68,7 +71,7 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { } /// The metadata of the database contains information about the version and its height. -#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { V1 { version: u32, @@ -77,7 +80,7 @@ pub enum DatabaseMetadata { V2 { version: u32, height: Height, - balances_indexation_progress: Height, + indexation_progress: HashMap, }, } @@ -99,13 +102,16 @@ impl DatabaseMetadata { } /// Returns the height of the database. - pub fn balances_indexation_progress(&self) -> Option<&Height> { + pub fn balances_indexation_progress( + &self, + indexation_type: IndexationType, + ) -> Option<&Height> { match self { Self::V1 { height, .. } => None, Self::V2 { - balances_indexation_progress, + indexation_progress, .. - } => Some(balances_indexation_progress), + } => indexation_progress.get(&indexation_type), } } } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 79b24bd7cd5..77506cc5a74 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -74,7 +74,7 @@ where let new_metadata = DatabaseMetadata::V2 { version: *version + 1, height: *height, - balances_indexation_progress: Default::default(), + indexation_progress: Default::default(), }; info!("Migrating metadata from V1 to version V2..."); dbg!(&new_metadata); @@ -84,11 +84,7 @@ where info!("...Migrated!"); Ok(()) } - DatabaseMetadata::V2 { - version, - height, - balances_indexation_progress, - } => return Ok(()), + DatabaseMetadata::V2 { .. } => return Ok(()), } } } From 7472db7975784781383f8980b428fa443e099521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 12:37:31 +0200 Subject: [PATCH 009/111] into_genesis() attemt --- crates/fuel-core/src/combined_database.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 9c83da03e3b..f286eec53ea 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -136,7 +136,14 @@ impl CombinedDatabase { } pub fn migrate_metadata(&mut self) -> StorageResult<()> { - self.off_chain.migrate_metadata()?; + // Error: Off chain database is already initialized + // let mut unchecked_off_chain = + // self.off_chain().clone().into_genesis().map_err(|_| { + // anyhow::anyhow!("Off chain database is already initialized") + // })?; + // unchecked_off_chain.migrate_metadata()?; + + self.migrate_metadata()?; Ok(()) } From b4d2e0f8be28392f36af36b5b1a756e6a409adeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 13:29:09 +0200 Subject: [PATCH 010/111] Add some TODOs with ideas for the future --- crates/fuel-core/src/graphql_api/storage/balances.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 24787d09970..311e4624de8 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -37,6 +37,7 @@ const BALANCES_KEY_SIZE: usize = Address::LEN + AssetId::LEN; type Amount = u64; +// TODO[RC]: Maybe use: macro_rules! double_key { pub type BalancesKey = [u8; Address::LEN + AssetId::LEN]; /// These table stores the balances of asset id per owner. @@ -213,3 +214,8 @@ mod tests { assert_eq!(expected, actual); } } + +// TODO[RC]: Reuse this to test basic functionality +// fuel_core_storage::basic_storage_tests!( +// +// then add an integration test to verify the logic of the balances \ No newline at end of file From d38fdb3baac43d7b9cee4c2f0133bf7172359c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 16:00:14 +0200 Subject: [PATCH 011/111] Use `double_key!` macro to define the balances key --- .../src/graphql_api/storage/balances.rs | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 311e4624de8..45abb77365b 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -26,6 +26,7 @@ use fuel_core_types::{ Bytes8, }, fuel_types::BlockHeight, + fuel_vm::double_key, services::txpool::TransactionStatus, }; use std::{ @@ -33,12 +34,9 @@ use std::{ mem::size_of, }; -const BALANCES_KEY_SIZE: usize = Address::LEN + AssetId::LEN; - type Amount = u64; -// TODO[RC]: Maybe use: macro_rules! double_key { -pub type BalancesKey = [u8; Address::LEN + AssetId::LEN]; +double_key!(BalancesKey, Address, address, AssetId, asset_id); /// These table stores the balances of asset id per owner. pub struct Balances; @@ -102,22 +100,14 @@ mod tests { let new_balance = current_balance.unwrap_or(0) + amount; let db = self.database.off_chain_mut(); - - let mut key = [0; Address::LEN + AssetId::LEN]; - key[0..Address::LEN].copy_from_slice(owner.as_ref()); - key[Address::LEN..].copy_from_slice(asset_id.as_ref()); - + let key = BalancesKey::new(owner, asset_id); let _ = StorageMutate::::insert(db, &key, &new_balance) .expect("couldn't store test asset"); } pub fn query_balance(&self, owner: &Address, asset_id: &AssetId) -> Option { let db = self.database.off_chain(); - - let mut key = [0; Address::LEN + AssetId::LEN]; - key[0..Address::LEN].copy_from_slice(owner.as_ref()); - key[Address::LEN..].copy_from_slice(asset_id.as_ref()); - + let key = BalancesKey::new(owner, asset_id); let result = StorageInspect::::get(db, &key).unwrap(); result.map(|r| r.into_owned()) @@ -129,11 +119,8 @@ mod tests { let mut key_prefix = owner.as_ref().to_vec(); db.entries::(Some(key_prefix), IterDirection::Forward) .map(|asset| { - let asset = asset.unwrap(); - let asset_id = - AssetId::from_bytes_ref_checked(&asset.key[AssetId::LEN..]) - .copied() - .expect("incorrect bytes"); + let asset = asset.expect("TODO[RC]: Fixme"); + let asset_id = asset.key.asset_id().clone(); let balance = asset.value; (asset_id, balance) }) @@ -218,4 +205,4 @@ mod tests { // TODO[RC]: Reuse this to test basic functionality // fuel_core_storage::basic_storage_tests!( // -// then add an integration test to verify the logic of the balances \ No newline at end of file +// then add an integration test to verify the logic of the balances From 292acab9b15531915bdc81e095437ea6831176cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 16:18:59 +0200 Subject: [PATCH 012/111] Add `basic_storage_tests!` for Balances --- .../src/graphql_api/storage/balances.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 45abb77365b..c7a21220f03 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -29,6 +29,11 @@ use fuel_core_types::{ fuel_vm::double_key, services::txpool::TransactionStatus, }; +use rand::{ + distributions::Standard, + prelude::Distribution, + Rng, +}; use std::{ array::TryFromSliceError, mem::size_of, @@ -37,6 +42,13 @@ use std::{ type Amount = u64; double_key!(BalancesKey, Address, address, AssetId, asset_id); +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> BalancesKey { + let mut bytes = [0u8; BalancesKey::LEN]; + rng.fill_bytes(bytes.as_mut()); + BalancesKey::from_array(bytes) + } +} /// These table stores the balances of asset id per owner. pub struct Balances; @@ -200,9 +212,10 @@ mod tests { let actual = db.query_balances(&carol); assert_eq!(expected, actual); } -} -// TODO[RC]: Reuse this to test basic functionality -// fuel_core_storage::basic_storage_tests!( -// -// then add an integration test to verify the logic of the balances + fuel_core_storage::basic_storage_tests!( + Balances, + ::Key::default(), + ::Value::default() + ); +} From 8342c8f6846d882da4d18dbb9b5a99625f39a3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 17 Oct 2024 17:16:39 +0200 Subject: [PATCH 013/111] Balances DB stores separate information for coins and messages --- .../src/graphql_api/storage/balances.rs | 175 ++++++++++++++---- 1 file changed, 139 insertions(+), 36 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index c7a21220f03..4656b8903bc 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -39,7 +39,13 @@ use std::{ mem::size_of, }; -type Amount = u64; +#[derive( + Debug, Default, Clone, Copy, serde::Serialize, serde::Deserialize, Eq, PartialEq, +)] +pub struct Amount { + coins: u64, + messages: u64, +} double_key!(BalancesKey, Address, address, AssetId, asset_id); impl Distribution for Standard { @@ -69,6 +75,7 @@ impl TableWithBlueprint for Balances { } } +// TODO[RC]: This needs to be additionally tested with a proper integration test #[cfg(test)] mod tests { use std::collections::HashMap; @@ -85,7 +92,10 @@ mod tests { Bytes8, }; - use crate::combined_database::CombinedDatabase; + use crate::{ + combined_database::CombinedDatabase, + graphql_api::storage::balances::Amount, + }; use super::{ Balances, @@ -103,13 +113,16 @@ mod tests { } } - pub fn balance_tx( + pub fn register_amount( &mut self, owner: &Address, - (asset_id, amount): &(AssetId, u64), + (asset_id, amount): &(AssetId, Amount), ) { let current_balance = self.query_balance(owner, asset_id); - let new_balance = current_balance.unwrap_or(0) + amount; + let new_balance = Amount { + coins: current_balance.unwrap_or_default().coins + amount.coins, + messages: current_balance.unwrap_or_default().messages + amount.messages, + }; let db = self.database.off_chain_mut(); let key = BalancesKey::new(owner, asset_id); @@ -117,7 +130,11 @@ mod tests { .expect("couldn't store test asset"); } - pub fn query_balance(&self, owner: &Address, asset_id: &AssetId) -> Option { + pub fn query_balance( + &self, + owner: &Address, + asset_id: &AssetId, + ) -> Option { let db = self.database.off_chain(); let key = BalancesKey::new(owner, asset_id); let result = StorageInspect::::get(db, &key).unwrap(); @@ -125,7 +142,7 @@ mod tests { result.map(|r| r.into_owned()) } - pub fn query_balances(&self, owner: &Address) -> HashMap { + pub fn query_balances(&self, owner: &Address) -> HashMap { let db = self.database.off_chain(); let mut key_prefix = owner.as_ref().to_vec(); @@ -151,25 +168,66 @@ mod tests { let ASSET_1 = AssetId::from([1; 32]); let ASSET_2 = AssetId::from([2; 32]); - // Alice has 100 of asset 1 and a total of 1000 of asset 2 - let alice_tx_1 = (ASSET_1, 100_u64); - let alice_tx_2 = (ASSET_2, 600_u64); - let alice_tx_3 = (ASSET_2, 400_u64); + let alice_tx_1 = ( + ASSET_1, + Amount { + coins: 100, + messages: 0, + }, + ); + let alice_tx_2 = ( + ASSET_2, + Amount { + coins: 600, + messages: 0, + }, + ); + let alice_tx_3 = ( + ASSET_2, + Amount { + coins: 400, + messages: 0, + }, + ); // Carol has 200 of asset 2 - let carol_tx_1 = (ASSET_2, 200_u64); - - let res = db.balance_tx(&alice, &alice_tx_1); - let res = db.balance_tx(&alice, &alice_tx_2); - let res = db.balance_tx(&alice, &alice_tx_3); - let res = db.balance_tx(&carol, &carol_tx_1); + let carol_tx_1 = ( + ASSET_2, + Amount { + coins: 200, + messages: 0, + }, + ); + + let res = db.register_amount(&alice, &alice_tx_1); + let res = db.register_amount(&alice, &alice_tx_2); + let res = db.register_amount(&alice, &alice_tx_3); + let res = db.register_amount(&carol, &carol_tx_1); // Alice has correct balances - assert_eq!(db.query_balance(&alice, &alice_tx_1.0), Some(100)); - assert_eq!(db.query_balance(&alice, &alice_tx_2.0), Some(1000)); + assert_eq!( + db.query_balance(&alice, &alice_tx_1.0), + Some(Amount { + coins: 100, + messages: 0 + }) + ); + assert_eq!( + db.query_balance(&alice, &alice_tx_2.0), + Some(Amount { + coins: 1000, + messages: 0 + }) + ); // Carol has correct balances - assert_eq!(db.query_balance(&carol, &carol_tx_1.0), Some(200_u64)); + assert_eq!( + db.query_balance(&carol, &carol_tx_1.0), + Some(Amount { + coins: 200, + messages: 0 + }) + ); } #[test] @@ -183,23 +241,60 @@ mod tests { let ASSET_1 = AssetId::from([1; 32]); let ASSET_2 = AssetId::from([2; 32]); - // Alice has 100 of asset 1 and a total of 1000 of asset 2 - let alice_tx_1 = (ASSET_1, 100_u64); - let alice_tx_2 = (ASSET_2, 600_u64); - let alice_tx_3 = (ASSET_2, 400_u64); - - // Carol has 200 of asset 2 - let carol_tx_1 = (ASSET_2, 200_u64); - - let res = db.balance_tx(&alice, &alice_tx_1); - let res = db.balance_tx(&alice, &alice_tx_2); - let res = db.balance_tx(&alice, &alice_tx_3); - let res = db.balance_tx(&carol, &carol_tx_1); + let alice_tx_1 = ( + ASSET_1, + Amount { + coins: 100, + messages: 0, + }, + ); + let alice_tx_2 = ( + ASSET_2, + Amount { + coins: 600, + messages: 0, + }, + ); + let alice_tx_3 = ( + ASSET_2, + Amount { + coins: 400, + messages: 0, + }, + ); + + let carol_tx_1 = ( + ASSET_2, + Amount { + coins: 200, + messages: 0, + }, + ); + + let res = db.register_amount(&alice, &alice_tx_1); + let res = db.register_amount(&alice, &alice_tx_2); + let res = db.register_amount(&alice, &alice_tx_3); + let res = db.register_amount(&carol, &carol_tx_1); // Verify Alice balances - let expected: HashMap<_, _> = vec![(ASSET_1, 100_u64), (ASSET_2, 1000_u64)] - .into_iter() - .collect(); + let expected: HashMap<_, _> = vec![ + ( + ASSET_1, + Amount { + coins: 100, + messages: 0, + }, + ), + ( + ASSET_2, + Amount { + coins: 1000, + messages: 0, + }, + ), + ] + .into_iter() + .collect(); let actual = db.query_balances(&alice); assert_eq!(expected, actual); @@ -208,7 +303,15 @@ mod tests { assert_eq!(HashMap::new(), actual); // Verify Carol balances - let expected: HashMap<_, _> = vec![(ASSET_2, 200_u64)].into_iter().collect(); + let expected: HashMap<_, _> = vec![( + ASSET_2, + Amount { + coins: 200, + messages: 0, + }, + )] + .into_iter() + .collect(); let actual = db.query_balances(&carol); assert_eq!(expected, actual); } From 9a9f120a55951fe0958bdfd35e4da8117d976463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 11:23:17 +0200 Subject: [PATCH 014/111] Fix the recursive call --- crates/fuel-core/src/combined_database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index f286eec53ea..b2e6c78c93e 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -143,7 +143,7 @@ impl CombinedDatabase { // })?; // unchecked_off_chain.migrate_metadata()?; - self.migrate_metadata()?; + self.off_chain.migrate_metadata()?; Ok(()) } From 6aa9325ddf3118b45072126b0c9650620046166e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 13:31:34 +0200 Subject: [PATCH 015/111] Init indexation progresses with 0 upon metadata migration --- crates/fuel-core/src/database/metadata.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 77506cc5a74..bce18c2ecbb 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -5,6 +5,7 @@ use crate::database::{ }, Database, Error as DatabaseError, + IndexationType, }; use fuel_core_storage::{ blueprint::plain::Plain, @@ -71,10 +72,14 @@ where match current_metadata.as_ref() { DatabaseMetadata::V1 { version, height } => { + let initial_progress = [ + (IndexationType::Balances, Description::Height::default()), + (IndexationType::CoinsToSpend, Description::Height::default()), + ]; let new_metadata = DatabaseMetadata::V2 { version: *version + 1, height: *height, - indexation_progress: Default::default(), + indexation_progress: initial_progress.into_iter().collect(), }; info!("Migrating metadata from V1 to version V2..."); dbg!(&new_metadata); From b153db48954740f9766f51cc61750a160c433ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 16:43:05 +0200 Subject: [PATCH 016/111] Remove debug prints --- crates/fuel-core/src/database.rs | 4 ---- crates/fuel-core/src/database/storage.rs | 2 -- 2 files changed, 6 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 9436d5f8ce5..5b2885551af 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -442,14 +442,10 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { - dbg!(&changes); - // Gets the all new heights from the `changes` let iterator = ChangesIterator::::new(&changes); let new_heights = heights_lookup(&iterator)?; - dbg!(&new_heights); - // Changes for each block should be committed separately. // If we have more than one height, it means we are mixing commits // for several heights in one batch - return error in this case. diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 9d13005eb82..e1809b1f181 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -43,7 +43,6 @@ where ); let prev = transaction.storage_as_mut::().replace(key, value)?; let changes = transaction.into_changes(); - dbg!(&changes); self.commit_changes_forced(changes)?; Ok(prev) } @@ -68,7 +67,6 @@ where ); let prev = transaction.storage_as_mut::().replace(key, value)?; let changes = transaction.into_changes(); - dbg!(&changes); self.commit_changes(changes)?; Ok(prev) } From 512a8a3fbf3f8ff63b29e1f29c69931b12d6f6c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 16:43:43 +0200 Subject: [PATCH 017/111] Store incoming balance in the new Balances DB --- crates/fuel-core/src/graphql_api/ports.rs | 2 ++ crates/fuel-core/src/graphql_api/storage.rs | 2 +- .../src/graphql_api/storage/balances.rs | 6 ++++ .../src/graphql_api/worker_service.rs | 28 ++++++++++++++++--- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 077a48d1637..793957efba2 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -273,6 +273,7 @@ pub mod worker { }, }, graphql_api::storage::{ + balances::Balances, da_compression::*, old::{ OldFuelBlockConsensus, @@ -327,6 +328,7 @@ pub mod worker { + StorageMutate + StorageMutate + StorageMutate + + StorageMutate + StorageMutate + StorageMutate + StorageMutate diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index a8b26685953..f398c20bab8 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -36,7 +36,7 @@ use fuel_core_types::{ }; use statistic::StatisticTable; -mod balances; +pub mod balances; pub mod blocks; pub mod coins; pub mod contracts; diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 4656b8903bc..ed0221856b7 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -47,6 +47,12 @@ pub struct Amount { messages: u64, } +impl Amount { + pub fn new(coins: u64, messages: u64) -> Self { + Self { coins, messages } + } +} + double_key!(BalancesKey, Address, address, AssetId, asset_id); impl Distribution for Standard { fn sample(&self, rng: &mut R) -> BalancesKey { diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 959733d4919..bf8cee42b23 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,9 +1,15 @@ use super::{ da_compression::da_compress_block, - storage::old::{ - OldFuelBlockConsensus, - OldFuelBlocks, - OldTransactions, + storage::{ + balances::{ + Amount, + BalancesKey, + }, + old::{ + OldFuelBlockConsensus, + OldFuelBlocks, + OldTransactions, + }, }, }; use crate::{ @@ -13,6 +19,7 @@ use crate::{ worker::OffChainDatabaseTransaction, }, storage::{ + balances::Balances, blocks::FuelBlockIdsToHeights, coins::{ owner_coin_id_key, @@ -94,6 +101,7 @@ use std::{ borrow::Cow, ops::Deref, }; +use tracing::info; #[cfg(test)] mod tests; @@ -216,6 +224,18 @@ where block_st_transaction .storage_as_mut::() .insert(&coin_by_owner, &())?; + + let address = coin.owner; + let asset_id = coin.asset_id; + let balances_key = BalancesKey::new(&address, &asset_id); + + // TODO[RC]: Do not overwrite values here, update them. + let amount = Amount::new(coin.amount, 0); + info!("XXX - inserting amount: {:?}...", amount); + block_st_transaction + .storage_as_mut::() + .insert(&balances_key, &amount); + info!("...inserted!"); } Event::CoinConsumed(coin) => { let key = owner_coin_id_key(&coin.owner, &coin.utxo_id); From adf9e2a4f315dcef6fcd1b3f832dd6f078c27425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 17:07:16 +0200 Subject: [PATCH 018/111] Read balance from the new Balances database --- crates/fuel-core/src/graphql_api/ports.rs | 2 ++ .../src/graphql_api/storage/balances.rs | 8 +++++++ crates/fuel-core/src/query/balance.rs | 5 +++++ .../service/adapters/graphql_api/off_chain.rs | 22 +++++++++++++++---- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 793957efba2..3b81dbf8b45 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -71,6 +71,8 @@ pub trait OffChainDatabase: Send + Sync { fn tx_status(&self, tx_id: &TxId) -> StorageResult; + fn balance(&self, owner: &Address, asset_id: &AssetId) -> StorageResult; + fn owned_coins_ids( &self, owner: &Address, diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index ed0221856b7..3ca0afc9f9b 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -51,6 +51,14 @@ impl Amount { pub fn new(coins: u64, messages: u64) -> Self { Self { coins, messages } } + + pub fn coins(&self) -> u64 { + self.coins + } + + pub fn messages(&self) -> u64 { + self.messages + } } double_key!(BalancesKey, Address, address, AssetId, asset_id); diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 161fd64b87e..1b1d723eba7 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -36,6 +36,7 @@ impl ReadView { asset_id: AssetId, base_asset_id: AssetId, ) -> StorageResult { + // The old way. let amount = AssetQuery::new( &owner, &AssetSpendTarget::new(asset_id, u64::MAX, u16::MAX), @@ -53,6 +54,10 @@ impl ReadView { }) .await?; + // The new way. + // TODO[RC]: balance could return both coins and messages + let amount = self.off_chain.balance(&owner, &asset_id)?; + Ok(AddressBalance { owner, amount, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index d554c7ddc45..b3323422c34 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -16,10 +16,16 @@ use crate::{ transactions::OwnedTransactionIndexCursor, }, }, - graphql_api::storage::old::{ - OldFuelBlockConsensus, - OldFuelBlocks, - OldTransactions, + graphql_api::storage::{ + balances::{ + Balances, + BalancesKey, + }, + old::{ + OldFuelBlockConsensus, + OldFuelBlocks, + OldTransactions, + }, }, }; use fuel_core_storage::{ @@ -51,6 +57,7 @@ use fuel_core_types::{ entities::relayer::transaction::RelayedTransactionStatus, fuel_tx::{ Address, + AssetId, Bytes32, ContractId, Salt, @@ -187,6 +194,13 @@ impl OffChainDatabase for OffChainIterableKeyValueView { fn message_is_spent(&self, nonce: &Nonce) -> StorageResult { self.message_is_spent(nonce) } + + fn balance(&self, owner: &Address, asset_id: &AssetId) -> StorageResult { + self.storage_as_ref::() + .get(&BalancesKey::new(owner, asset_id))? + .map(|amount| amount.coins()) + .ok_or(not_found!(Balances)) + } } impl worker::OffChainDatabase for Database { From 344ca905b9feb83568f1fd37959506164eb5b216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 18 Oct 2024 17:12:35 +0200 Subject: [PATCH 019/111] Update coin balance, don't overwrite --- crates/fuel-core/src/graphql_api/worker_service.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index bf8cee42b23..1f8fec4859c 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -229,9 +229,16 @@ where let asset_id = coin.asset_id; let balances_key = BalancesKey::new(&address, &asset_id); - // TODO[RC]: Do not overwrite values here, update them. - let amount = Amount::new(coin.amount, 0); - info!("XXX - inserting amount: {:?}...", amount); + // TODO[RC]: Use some separate, testable function for this and also take care of "messages" + let current_amount = block_st_transaction + .storage::() + .get(&balances_key)? + .map(|amount| amount.coins()) + .unwrap_or_default(); + + let amount = Amount::new(current_amount.saturating_add(coin.amount), 0); + + info!("XXX - current amount: {current_amount}, new amount: {}, total new amount: {:?}...", coin.amount, amount); block_st_transaction .storage_as_mut::() .insert(&balances_key, &amount); From f73dbed38cca2f8e6b4f4cf7d976c42ec8279c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 10:39:32 +0200 Subject: [PATCH 020/111] Use more detailed `IndexationStatus`, not just block height --- crates/fuel-core/src/database.rs | 17 +++++++++++++++++ .../src/database/database_description.rs | 11 +++++++---- crates/fuel-core/src/database/metadata.rs | 5 +++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 5b2885551af..40ce7abb9c5 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -104,6 +104,23 @@ pub(crate) enum IndexationType { CoinsToSpend, } +#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) enum IndexationStatus { + Pending, + CompletedUntil(BlockHeight), + Finished, +} + +impl IndexationStatus { + pub fn new() -> Self { + IndexationStatus::Pending + } + + pub fn is_finished(&self) -> bool { + matches!(self, IndexationStatus::Finished) + } +} + #[derive(Default, Debug, Copy, Clone)] pub struct GenesisStage; diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 548887fec48..74f49c12994 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -6,7 +6,10 @@ use fuel_core_types::{ }; use std::collections::HashMap; -use super::IndexationType; +use super::{ + IndexationStatus, + IndexationType, +}; pub mod gas_price; pub mod off_chain; @@ -80,7 +83,7 @@ pub enum DatabaseMetadata { V2 { version: u32, height: Height, - indexation_progress: HashMap, + indexation_progress: HashMap, }, } @@ -101,11 +104,11 @@ impl DatabaseMetadata { } } - /// Returns the height of the database. + /// Returns the indexation progress of a database pub fn balances_indexation_progress( &self, indexation_type: IndexationType, - ) -> Option<&Height> { + ) -> Option<&IndexationStatus> { match self { Self::V1 { height, .. } => None, Self::V2 { diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index bce18c2ecbb..8dc6ee9dedb 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -5,6 +5,7 @@ use crate::database::{ }, Database, Error as DatabaseError, + IndexationStatus, IndexationType, }; use fuel_core_storage::{ @@ -73,8 +74,8 @@ where match current_metadata.as_ref() { DatabaseMetadata::V1 { version, height } => { let initial_progress = [ - (IndexationType::Balances, Description::Height::default()), - (IndexationType::CoinsToSpend, Description::Height::default()), + (IndexationType::Balances, IndexationStatus::new()), + (IndexationType::CoinsToSpend, IndexationStatus::new()), ]; let new_metadata = DatabaseMetadata::V2 { version: *version + 1, From 80da09bbae5a9b1ee61bdfa370fa7b68d6606cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 13:35:51 +0200 Subject: [PATCH 021/111] Add processing of `MessageImported` --- .../src/graphql_api/storage/balances.rs | 30 ++++- .../src/graphql_api/worker_service.rs | 81 ++++++++++++- .../service/adapters/graphql_api/off_chain.rs | 28 ++++- .../service/genesis/importer/import_task.rs | 1 + .../src/service/genesis/importer/off_chain.rs | 15 ++- tests/tests/balances.rs | 108 ++++++++++++++++++ 6 files changed, 251 insertions(+), 12 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 3ca0afc9f9b..0a77c41c0d3 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -39,6 +39,9 @@ use std::{ mem::size_of, }; +// TODO[RC]: Do not split to coins and messages here, just leave "amount". +// amount for coins = owner+asset_id +// amount for messages = owner+base_asset_id #[derive( Debug, Default, Clone, Copy, serde::Serialize, serde::Deserialize, Eq, PartialEq, )] @@ -47,9 +50,19 @@ pub struct Amount { messages: u64, } +impl core::fmt::Display for Amount { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "coins: {}, messages: {}", self.coins, self.messages) + } +} + impl Amount { - pub fn new(coins: u64, messages: u64) -> Self { - Self { coins, messages } + pub fn new_coins(coins: u64) -> Self { + Self { coins, messages: 0 } + } + + pub fn new_messages(messages: u64) -> Self { + Self { coins: 0, messages } } pub fn coins(&self) -> u64 { @@ -59,6 +72,19 @@ impl Amount { pub fn messages(&self) -> u64 { self.messages } + + pub fn saturating_add(&self, other: &Self) -> Self { + Self { + coins: self + .coins + .checked_add(other.coins) + .expect("TODO[RC]: balance too large"), + messages: self + .messages + .checked_add(other.messages) + .expect("TODO[RC]: balance too large"), + } + } } double_key!(BalancesKey, Address, address, AssetId, asset_id); diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 1f8fec4859c..4de143d75ba 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,4 +1,5 @@ use super::{ + api_service::ConsensusProvider, da_compression::da_compress_block, storage::{ balances::{ @@ -34,6 +35,7 @@ use crate::{ }, }, graphql_api::storage::relayed_transactions::RelayedTransactionStatuses, + service::adapters::ConsensusParametersProvider, }; use fuel_core_metrics::graphql_metrics::graphql_metrics; use fuel_core_services::{ @@ -45,6 +47,12 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ + iter::{ + IterDirection, + IteratorOverTable, + }, + not_found, + tables::ConsensusParametersVersions, Error as StorageError, Result as StorageResult, StorageAsMut, @@ -69,6 +77,7 @@ use fuel_core_types::{ CoinPredicate, CoinSigned, }, + AssetId, Contract, Input, Output, @@ -97,6 +106,7 @@ use futures::{ FutureExt, StreamExt, }; +use hex::FromHex; use std::{ borrow::Cow, ops::Deref, @@ -131,6 +141,7 @@ pub struct Task { block_importer: BoxStream, database: D, chain_id: ChainId, + base_asset_id: AssetId, da_compression_config: DaCompressionConfig, continue_on_error: bool, } @@ -165,6 +176,7 @@ where process_executor_events( result.events.iter().map(Cow::Borrowed), &mut transaction, + &self.base_asset_id, )?; match self.da_compression_config { @@ -193,6 +205,7 @@ where pub fn process_executor_events<'a, Iter, T>( events: Iter, block_st_transaction: &mut T, + base_asset_id: &AssetId, ) -> anyhow::Result<()> where Iter: Iterator>, @@ -201,12 +214,43 @@ where for event in events { match event.deref() { Event::MessageImported(message) => { + // *** "Old" behavior *** block_st_transaction .storage_as_mut::() .insert( &OwnedMessageKey::new(message.recipient(), message.nonce()), &(), )?; + + // *** "New" behavior (using Balances DB) *** + let address = message.recipient(); + let asset_id = base_asset_id; + let balances_key = BalancesKey::new(&address, &asset_id); + + // TODO[RC]: Use some separate, testable function for this and also take care of "messages" + let amount = block_st_transaction + .storage::() + .get(&balances_key)? + .unwrap_or_default(); + + let new_amount = + amount.saturating_add(&Amount::new_messages(message.amount())); + + println!( + "Processing message with amount: {} (asset_id={})", + message.amount(), + asset_id + ); + println!( + "Storing {new_amount} for message under key [{},{:?}]", + address, asset_id + ); + + info!("XXX - current amount: {amount}, adding {} messages, new amount: {new_amount}", message.amount()); + block_st_transaction + .storage_as_mut::() + .insert(&balances_key, &new_amount); // TODO[RC]: .replace() + info!("...inserted!"); } Event::MessageConsumed(message) => { block_st_transaction @@ -220,35 +264,49 @@ where .insert(message.nonce(), &())?; } Event::CoinCreated(coin) => { + // *** "Old" behavior *** let coin_by_owner = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .insert(&coin_by_owner, &())?; + // *** "New" behavior (using Balances DB) *** let address = coin.owner; let asset_id = coin.asset_id; let balances_key = BalancesKey::new(&address, &asset_id); // TODO[RC]: Use some separate, testable function for this and also take care of "messages" - let current_amount = block_st_transaction + let amount = block_st_transaction .storage::() .get(&balances_key)? - .map(|amount| amount.coins()) .unwrap_or_default(); - let amount = Amount::new(current_amount.saturating_add(coin.amount), 0); + let new_amount = amount.saturating_add(&Amount::new_coins(coin.amount)); + + println!( + "Processing coin with amount: {} (asset_id={})", + coin.amount, asset_id + ); + println!( + "Storing {new_amount} for coin under key [{},{:?}]", + address, asset_id + ); - info!("XXX - current amount: {current_amount}, new amount: {}, total new amount: {:?}...", coin.amount, amount); + info!("XXX - current amount: {amount}, adding {} coins, new amount: {new_amount}", coin.amount); block_st_transaction .storage_as_mut::() - .insert(&balances_key, &amount); + .insert(&balances_key, &new_amount); // TODO[RC]: .replace() info!("...inserted!"); } Event::CoinConsumed(coin) => { + // *** "Old" behavior *** let key = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .remove(&key)?; + + // *** "New" behavior (using Balances DB) *** + } Event::ForcedTransactionFailed { id, @@ -471,6 +529,16 @@ where Ok(()) } +fn base_asset_id() -> AssetId { + // TODO[RC]: This is just a hack, get base asset id from consensus parameters here. + let base_asset_id = + Vec::from_hex("0000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let arr: [u8; 32] = base_asset_id.try_into().unwrap(); + let base_asset_id = AssetId::new(arr); + base_asset_id +} + #[async_trait::async_trait] impl RunnableService for InitializeTask @@ -500,6 +568,8 @@ where graphql_metrics().total_txs_count.set(total_tx_count as i64); } + let base_asset_id = base_asset_id(); + let InitializeTask { chain_id, da_compression_config, @@ -518,6 +588,7 @@ where chain_id, da_compression_config, continue_on_error, + base_asset_id, }; let mut target_chain_height = on_chain_database.latest_height()?; diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index b3323422c34..cb3d718a9ac 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -72,6 +72,17 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; +use hex::FromHex; + +fn base_asset_id() -> AssetId { + // TODO[RC]: This is just a hack, get base asset id from consensus parameters here. + let base_asset_id = + Vec::from_hex("0000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let arr: [u8; 32] = base_asset_id.try_into().unwrap(); + let base_asset_id = AssetId::new(arr); + base_asset_id +} impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { @@ -196,10 +207,23 @@ impl OffChainDatabase for OffChainIterableKeyValueView { } fn balance(&self, owner: &Address, asset_id: &AssetId) -> StorageResult { - self.storage_as_ref::() + let coins = self + .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? .map(|amount| amount.coins()) - .ok_or(not_found!(Balances)) + .ok_or(not_found!(Balances))?; + + let base_asset_id = base_asset_id(); + + let messages = self + .storage_as_ref::() + .get(&BalancesKey::new(owner, &base_asset_id))? + .map(|amount| amount.messages()) + .ok_or(not_found!(Balances))?; + + Ok(coins + .checked_add(messages) + .expect("TODO[RC]: balance too big")) } } diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index 5cdcf636c8c..bff148964d4 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -11,6 +11,7 @@ use fuel_core_storage::{ StorageInspect, StorageMutate, }; +use fuel_core_types::fuel_tx::AssetId; use crate::{ database::{ diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index eef13bf9ee5..54894d78708 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -35,7 +35,10 @@ use fuel_core_storage::{ transactional::StorageTransaction, StorageAsMut, }; -use fuel_core_types::services::executor::Event; +use fuel_core_types::{ + fuel_tx::AssetId, + services::executor::Event, +}; use std::borrow::Cow; use super::{ @@ -110,7 +113,10 @@ impl ImportTable for Handler { let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - worker_service::process_executor_events(events, tx)?; + + let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here + + worker_service::process_executor_events(events, tx, &base_asset_id)?; Ok(()) } } @@ -128,7 +134,10 @@ impl ImportTable for Handler { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - worker_service::process_executor_events(events, tx)?; + + let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here + + worker_service::process_executor_events(events, tx, &base_asset_id)?; Ok(()) } } diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index a5892b434eb..94d882f9fff 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -32,6 +32,7 @@ use fuel_core_types::{ TransactionBuilder, }, }; +use hex::FromHex; #[tokio::test] async fn balance() { @@ -229,3 +230,110 @@ async fn first_5_balances() { assert_eq!(balances[i].amount, 300); } } + +#[tokio::test] +async fn foo_1() { + let owner = Address::default(); + + // TODO[RC]: This is just a hack + let asset_id = + Vec::from_hex("aabbccdd3d7fa5b672b530cbb84fcccb4ff8dc40f8176ef4544ddb1f1952ad07") + .unwrap(); + let arr: [u8; 32] = asset_id.try_into().unwrap(); + let asset_id = AssetId::new(arr); + + // let asset_id = AssetId::BASE; + + // setup config + let mut coin_generator = CoinConfigGenerator::new(); + let state_config = StateConfig { + contracts: vec![], + coins: vec![ + (owner, 50, asset_id), + (owner, 100, asset_id), + (owner, 150, asset_id), + ] + .into_iter() + .map(|(owner, amount, asset_id)| CoinConfig { + owner, + amount, + asset_id, + ..coin_generator.generate() + }) + .collect(), + messages: vec![(owner, 60), (owner, 90)] + .into_iter() + .enumerate() + .map(|(nonce, (owner, amount))| MessageConfig { + sender: owner, + recipient: owner, + nonce: (nonce as u64).into(), + amount, + data: vec![], + da_height: DaBlockHeight::from(0usize), + }) + .collect(), + ..Default::default() + }; + let config = Config::local_node_with_state_config(state_config); + + // setup server & client + let srv = FuelService::new_node(config).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + + // run test + let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); + assert_eq!(balance, 450); + + // spend some coins and check again + // let coins_per_asset = client + // .coins_to_spend(&owner, vec![(asset_id, 1, None)], None) + // .await + // .unwrap(); + + // let mut tx = TransactionBuilder::script(vec![], vec![]) + // .script_gas_limit(1_000_000) + // .to_owned(); + // for coins in coins_per_asset { + // for coin in coins { + // match coin { + // CoinType::Coin(coin) => tx.add_input(Input::coin_signed( + // coin.utxo_id, + // coin.owner, + // coin.amount, + // coin.asset_id, + // Default::default(), + // 0, + // )), + // CoinType::MessageCoin(message) => { + // tx.add_input(Input::message_coin_signed( + // message.sender, + // message.recipient, + // message.amount, + // message.nonce, + // 0, + // )) + // } + // CoinType::Unknown => panic!("Unknown coin"), + // }; + // } + // } + // let tx = tx + // .add_output(Output::Coin { + // to: Address::new([1u8; 32]), + // amount: 1, + // asset_id, + // }) + // .add_output(Output::Change { + // to: owner, + // amount: 0, + // asset_id, + // }) + // .add_witness(Default::default()) + // .finalize_as_transaction(); + + // client.submit_and_await_commit(&tx).await.unwrap(); + + // let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); + // assert_eq!(balance, 449); +} From ad5216dc6a30d34f03ebaf28be01d354cdbbaffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 13:42:25 +0200 Subject: [PATCH 022/111] Simplify processing of coins and message amounts --- .../src/graphql_api/storage/balances.rs | 541 ++++++++---------- .../src/graphql_api/worker_service.rs | 6 +- .../service/adapters/graphql_api/off_chain.rs | 4 +- 3 files changed, 251 insertions(+), 300 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 0a77c41c0d3..876397f1f0c 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -39,53 +39,7 @@ use std::{ mem::size_of, }; -// TODO[RC]: Do not split to coins and messages here, just leave "amount". -// amount for coins = owner+asset_id -// amount for messages = owner+base_asset_id -#[derive( - Debug, Default, Clone, Copy, serde::Serialize, serde::Deserialize, Eq, PartialEq, -)] -pub struct Amount { - coins: u64, - messages: u64, -} - -impl core::fmt::Display for Amount { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - write!(f, "coins: {}, messages: {}", self.coins, self.messages) - } -} - -impl Amount { - pub fn new_coins(coins: u64) -> Self { - Self { coins, messages: 0 } - } - - pub fn new_messages(messages: u64) -> Self { - Self { coins: 0, messages } - } - - pub fn coins(&self) -> u64 { - self.coins - } - - pub fn messages(&self) -> u64 { - self.messages - } - - pub fn saturating_add(&self, other: &Self) -> Self { - Self { - coins: self - .coins - .checked_add(other.coins) - .expect("TODO[RC]: balance too large"), - messages: self - .messages - .checked_add(other.messages) - .expect("TODO[RC]: balance too large"), - } - } -} +pub type Amount = u64; double_key!(BalancesKey, Address, address, AssetId, asset_id); impl Distribution for Standard { @@ -115,250 +69,251 @@ impl TableWithBlueprint for Balances { } } +// TODO[RC]: These are most likely not needed, we're testing at integration level. // TODO[RC]: This needs to be additionally tested with a proper integration test -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use fuel_core_storage::{ - iter::IterDirection, - StorageInspect, - StorageMutate, - }; - use fuel_core_types::fuel_tx::{ - Address, - AssetId, - Bytes64, - Bytes8, - }; - - use crate::{ - combined_database::CombinedDatabase, - graphql_api::storage::balances::Amount, - }; - - use super::{ - Balances, - BalancesKey, - }; - - pub struct TestDatabase { - database: CombinedDatabase, - } - - impl TestDatabase { - pub fn new() -> Self { - Self { - database: Default::default(), - } - } - - pub fn register_amount( - &mut self, - owner: &Address, - (asset_id, amount): &(AssetId, Amount), - ) { - let current_balance = self.query_balance(owner, asset_id); - let new_balance = Amount { - coins: current_balance.unwrap_or_default().coins + amount.coins, - messages: current_balance.unwrap_or_default().messages + amount.messages, - }; - - let db = self.database.off_chain_mut(); - let key = BalancesKey::new(owner, asset_id); - let _ = StorageMutate::::insert(db, &key, &new_balance) - .expect("couldn't store test asset"); - } - - pub fn query_balance( - &self, - owner: &Address, - asset_id: &AssetId, - ) -> Option { - let db = self.database.off_chain(); - let key = BalancesKey::new(owner, asset_id); - let result = StorageInspect::::get(db, &key).unwrap(); - - result.map(|r| r.into_owned()) - } - - pub fn query_balances(&self, owner: &Address) -> HashMap { - let db = self.database.off_chain(); - - let mut key_prefix = owner.as_ref().to_vec(); - db.entries::(Some(key_prefix), IterDirection::Forward) - .map(|asset| { - let asset = asset.expect("TODO[RC]: Fixme"); - let asset_id = asset.key.asset_id().clone(); - let balance = asset.value; - (asset_id, balance) - }) - .collect() - } - } - - #[test] - fn can_retrieve_balance_of_asset() { - let mut db = TestDatabase::new(); - - let alice = Address::from([1; 32]); - let bob = Address::from([2; 32]); - let carol = Address::from([3; 32]); - - let ASSET_1 = AssetId::from([1; 32]); - let ASSET_2 = AssetId::from([2; 32]); - - let alice_tx_1 = ( - ASSET_1, - Amount { - coins: 100, - messages: 0, - }, - ); - let alice_tx_2 = ( - ASSET_2, - Amount { - coins: 600, - messages: 0, - }, - ); - let alice_tx_3 = ( - ASSET_2, - Amount { - coins: 400, - messages: 0, - }, - ); - - // Carol has 200 of asset 2 - let carol_tx_1 = ( - ASSET_2, - Amount { - coins: 200, - messages: 0, - }, - ); - - let res = db.register_amount(&alice, &alice_tx_1); - let res = db.register_amount(&alice, &alice_tx_2); - let res = db.register_amount(&alice, &alice_tx_3); - let res = db.register_amount(&carol, &carol_tx_1); - - // Alice has correct balances - assert_eq!( - db.query_balance(&alice, &alice_tx_1.0), - Some(Amount { - coins: 100, - messages: 0 - }) - ); - assert_eq!( - db.query_balance(&alice, &alice_tx_2.0), - Some(Amount { - coins: 1000, - messages: 0 - }) - ); - - // Carol has correct balances - assert_eq!( - db.query_balance(&carol, &carol_tx_1.0), - Some(Amount { - coins: 200, - messages: 0 - }) - ); - } - - #[test] - fn can_retrieve_balances_of_all_assets_of_owner() { - let mut db = TestDatabase::new(); - - let alice = Address::from([1; 32]); - let bob = Address::from([2; 32]); - let carol = Address::from([3; 32]); - - let ASSET_1 = AssetId::from([1; 32]); - let ASSET_2 = AssetId::from([2; 32]); - - let alice_tx_1 = ( - ASSET_1, - Amount { - coins: 100, - messages: 0, - }, - ); - let alice_tx_2 = ( - ASSET_2, - Amount { - coins: 600, - messages: 0, - }, - ); - let alice_tx_3 = ( - ASSET_2, - Amount { - coins: 400, - messages: 0, - }, - ); - - let carol_tx_1 = ( - ASSET_2, - Amount { - coins: 200, - messages: 0, - }, - ); - - let res = db.register_amount(&alice, &alice_tx_1); - let res = db.register_amount(&alice, &alice_tx_2); - let res = db.register_amount(&alice, &alice_tx_3); - let res = db.register_amount(&carol, &carol_tx_1); - - // Verify Alice balances - let expected: HashMap<_, _> = vec![ - ( - ASSET_1, - Amount { - coins: 100, - messages: 0, - }, - ), - ( - ASSET_2, - Amount { - coins: 1000, - messages: 0, - }, - ), - ] - .into_iter() - .collect(); - let actual = db.query_balances(&alice); - assert_eq!(expected, actual); - - // Verify Bob balances - let actual = db.query_balances(&bob); - assert_eq!(HashMap::new(), actual); - - // Verify Carol balances - let expected: HashMap<_, _> = vec![( - ASSET_2, - Amount { - coins: 200, - messages: 0, - }, - )] - .into_iter() - .collect(); - let actual = db.query_balances(&carol); - assert_eq!(expected, actual); - } - - fuel_core_storage::basic_storage_tests!( - Balances, - ::Key::default(), - ::Value::default() - ); -} +// #[cfg(test)] +// mod tests { +// use std::collections::HashMap; +// +// use fuel_core_storage::{ +// iter::IterDirection, +// StorageInspect, +// StorageMutate, +// }; +// use fuel_core_types::fuel_tx::{ +// Address, +// AssetId, +// Bytes64, +// Bytes8, +// }; +// +// use crate::{ +// combined_database::CombinedDatabase, +// graphql_api::storage::balances::Amount, +// }; +// +// use super::{ +// Balances, +// BalancesKey, +// }; +// +// pub struct TestDatabase { +// database: CombinedDatabase, +// } +// +// impl TestDatabase { +// pub fn new() -> Self { +// Self { +// database: Default::default(), +// } +// } +// +// pub fn register_amount( +// &mut self, +// owner: &Address, +// (asset_id, amount): &(AssetId, Amount), +// ) { +// let current_balance = self.query_balance(owner, asset_id); +// let new_balance = Amount { +// coins: current_balance.unwrap_or_default().coins + amount.coins, +// messages: current_balance.unwrap_or_default().messages + amount.messages, +// }; +// +// let db = self.database.off_chain_mut(); +// let key = BalancesKey::new(owner, asset_id); +// let _ = StorageMutate::::insert(db, &key, &new_balance) +// .expect("couldn't store test asset"); +// } +// +// pub fn query_balance( +// &self, +// owner: &Address, +// asset_id: &AssetId, +// ) -> Option { +// let db = self.database.off_chain(); +// let key = BalancesKey::new(owner, asset_id); +// let result = StorageInspect::::get(db, &key).unwrap(); +// +// result.map(|r| r.into_owned()) +// } +// +// pub fn query_balances(&self, owner: &Address) -> HashMap { +// let db = self.database.off_chain(); +// +// let mut key_prefix = owner.as_ref().to_vec(); +// db.entries::(Some(key_prefix), IterDirection::Forward) +// .map(|asset| { +// let asset = asset.expect("TODO[RC]: Fixme"); +// let asset_id = asset.key.asset_id().clone(); +// let balance = asset.value; +// (asset_id, balance) +// }) +// .collect() +// } +// } +// +// #[test] +// fn can_retrieve_balance_of_asset() { +// let mut db = TestDatabase::new(); +// +// let alice = Address::from([1; 32]); +// let bob = Address::from([2; 32]); +// let carol = Address::from([3; 32]); +// +// let ASSET_1 = AssetId::from([1; 32]); +// let ASSET_2 = AssetId::from([2; 32]); +// +// let alice_tx_1 = ( +// ASSET_1, +// Amount { +// coins: 100, +// messages: 0, +// }, +// ); +// let alice_tx_2 = ( +// ASSET_2, +// Amount { +// coins: 600, +// messages: 0, +// }, +// ); +// let alice_tx_3 = ( +// ASSET_2, +// Amount { +// coins: 400, +// messages: 0, +// }, +// ); +// +// Carol has 200 of asset 2 +// let carol_tx_1 = ( +// ASSET_2, +// Amount { +// coins: 200, +// messages: 0, +// }, +// ); +// +// let res = db.register_amount(&alice, &alice_tx_1); +// let res = db.register_amount(&alice, &alice_tx_2); +// let res = db.register_amount(&alice, &alice_tx_3); +// let res = db.register_amount(&carol, &carol_tx_1); +// +// Alice has correct balances +// assert_eq!( +// db.query_balance(&alice, &alice_tx_1.0), +// Some(Amount { +// coins: 100, +// messages: 0 +// }) +// ); +// assert_eq!( +// db.query_balance(&alice, &alice_tx_2.0), +// Some(Amount { +// coins: 1000, +// messages: 0 +// }) +// ); +// +// Carol has correct balances +// assert_eq!( +// db.query_balance(&carol, &carol_tx_1.0), +// Some(Amount { +// coins: 200, +// messages: 0 +// }) +// ); +// } +// +// #[test] +// fn can_retrieve_balances_of_all_assets_of_owner() { +// let mut db = TestDatabase::new(); +// +// let alice = Address::from([1; 32]); +// let bob = Address::from([2; 32]); +// let carol = Address::from([3; 32]); +// +// let ASSET_1 = AssetId::from([1; 32]); +// let ASSET_2 = AssetId::from([2; 32]); +// +// let alice_tx_1 = ( +// ASSET_1, +// Amount { +// coins: 100, +// messages: 0, +// }, +// ); +// let alice_tx_2 = ( +// ASSET_2, +// Amount { +// coins: 600, +// messages: 0, +// }, +// ); +// let alice_tx_3 = ( +// ASSET_2, +// Amount { +// coins: 400, +// messages: 0, +// }, +// ); +// +// let carol_tx_1 = ( +// ASSET_2, +// Amount { +// coins: 200, +// messages: 0, +// }, +// ); +// +// let res = db.register_amount(&alice, &alice_tx_1); +// let res = db.register_amount(&alice, &alice_tx_2); +// let res = db.register_amount(&alice, &alice_tx_3); +// let res = db.register_amount(&carol, &carol_tx_1); +// +// Verify Alice balances +// let expected: HashMap<_, _> = vec![ +// ( +// ASSET_1, +// Amount { +// coins: 100, +// messages: 0, +// }, +// ), +// ( +// ASSET_2, +// Amount { +// coins: 1000, +// messages: 0, +// }, +// ), +// ] +// .into_iter() +// .collect(); +// let actual = db.query_balances(&alice); +// assert_eq!(expected, actual); +// +// Verify Bob balances +// let actual = db.query_balances(&bob); +// assert_eq!(HashMap::new(), actual); +// +// Verify Carol balances +// let expected: HashMap<_, _> = vec![( +// ASSET_2, +// Amount { +// coins: 200, +// messages: 0, +// }, +// )] +// .into_iter() +// .collect(); +// let actual = db.query_balances(&carol); +// assert_eq!(expected, actual); +// } +// +// fuel_core_storage::basic_storage_tests!( +// Balances, +// ::Key::default(), +// ::Value::default() +// ); +// } diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 4de143d75ba..ff306c401ed 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -233,8 +233,7 @@ where .get(&balances_key)? .unwrap_or_default(); - let new_amount = - amount.saturating_add(&Amount::new_messages(message.amount())); + let new_amount = amount.saturating_add(message.amount()); println!( "Processing message with amount: {} (asset_id={})", @@ -281,7 +280,7 @@ where .get(&balances_key)? .unwrap_or_default(); - let new_amount = amount.saturating_add(&Amount::new_coins(coin.amount)); + let new_amount = amount.saturating_add(coin.amount); println!( "Processing coin with amount: {} (asset_id={})", @@ -306,7 +305,6 @@ where .remove(&key)?; // *** "New" behavior (using Balances DB) *** - } Event::ForcedTransactionFailed { id, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index cb3d718a9ac..28e8e17db2e 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -210,7 +210,6 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let coins = self .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? - .map(|amount| amount.coins()) .ok_or(not_found!(Balances))?; let base_asset_id = base_asset_id(); @@ -218,11 +217,10 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let messages = self .storage_as_ref::() .get(&BalancesKey::new(owner, &base_asset_id))? - .map(|amount| amount.messages()) .ok_or(not_found!(Balances))?; Ok(coins - .checked_add(messages) + .checked_add(*messages) .expect("TODO[RC]: balance too big")) } } From c784e44697ed29fdcae29b87d0f2aa7b2764dd65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 14:08:38 +0200 Subject: [PATCH 023/111] Extract `increase_balance()` --- .../src/graphql_api/worker_service.rs | 84 +++++++------------ 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index ff306c401ed..90d56b2a4f9 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -77,6 +77,7 @@ use fuel_core_types::{ CoinPredicate, CoinSigned, }, + Address, AssetId, Contract, Input, @@ -201,6 +202,26 @@ where } } +fn increase_balance( + owner: &Address, + asset_id: &AssetId, + amount: Amount, + tx: &mut T, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + // TODO[RC]: Make sure this operation is atomic + let balances_key = BalancesKey::new(owner, asset_id); + let current_balance = tx + .storage::() + .get(&balances_key)? + .unwrap_or_default(); + let new_balance = current_balance.saturating_add(amount); + tx.storage_as_mut::() + .insert(&balances_key, &new_balance) +} + /// Process the executor events and update the indexes for the messages and coins. pub fn process_executor_events<'a, Iter, T>( events: Iter, @@ -223,33 +244,12 @@ where )?; // *** "New" behavior (using Balances DB) *** - let address = message.recipient(); - let asset_id = base_asset_id; - let balances_key = BalancesKey::new(&address, &asset_id); - - // TODO[RC]: Use some separate, testable function for this and also take care of "messages" - let amount = block_st_transaction - .storage::() - .get(&balances_key)? - .unwrap_or_default(); - - let new_amount = amount.saturating_add(message.amount()); - - println!( - "Processing message with amount: {} (asset_id={})", + increase_balance( + &message.recipient(), + base_asset_id, message.amount(), - asset_id - ); - println!( - "Storing {new_amount} for message under key [{},{:?}]", - address, asset_id - ); - - info!("XXX - current amount: {amount}, adding {} messages, new amount: {new_amount}", message.amount()); - block_st_transaction - .storage_as_mut::() - .insert(&balances_key, &new_amount); // TODO[RC]: .replace() - info!("...inserted!"); + block_st_transaction, + )?; } Event::MessageConsumed(message) => { block_st_transaction @@ -270,32 +270,12 @@ where .insert(&coin_by_owner, &())?; // *** "New" behavior (using Balances DB) *** - let address = coin.owner; - let asset_id = coin.asset_id; - let balances_key = BalancesKey::new(&address, &asset_id); - - // TODO[RC]: Use some separate, testable function for this and also take care of "messages" - let amount = block_st_transaction - .storage::() - .get(&balances_key)? - .unwrap_or_default(); - - let new_amount = amount.saturating_add(coin.amount); - - println!( - "Processing coin with amount: {} (asset_id={})", - coin.amount, asset_id - ); - println!( - "Storing {new_amount} for coin under key [{},{:?}]", - address, asset_id - ); - - info!("XXX - current amount: {amount}, adding {} coins, new amount: {new_amount}", coin.amount); - block_st_transaction - .storage_as_mut::() - .insert(&balances_key, &new_amount); // TODO[RC]: .replace() - info!("...inserted!"); + increase_balance( + &coin.owner, + &coin.asset_id, + coin.amount, + block_st_transaction, + )?; } Event::CoinConsumed(coin) => { // *** "Old" behavior *** From 576266599e0e4808ebd1c50da22bc4c7b2acff43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 14:59:08 +0200 Subject: [PATCH 024/111] Store coin and message balances separately --- crates/fuel-core/src/graphql_api/ports.rs | 3 +- crates/fuel-core/src/graphql_api/storage.rs | 1 + .../src/graphql_api/storage/balances.rs | 19 ++++++ .../src/graphql_api/worker_service.rs | 49 +++++++++++--- .../service/adapters/graphql_api/off_chain.rs | 12 +++- tests/tests/balances.rs | 65 ++++++++++++++++--- 6 files changed, 127 insertions(+), 22 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 3b81dbf8b45..98ce4e6a779 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -275,7 +275,7 @@ pub mod worker { }, }, graphql_api::storage::{ - balances::Balances, + balances::{Balances, MessageBalances}, da_compression::*, old::{ OldFuelBlockConsensus, @@ -331,6 +331,7 @@ pub mod worker { + StorageMutate + StorageMutate + StorageMutate + + StorageMutate + StorageMutate + StorageMutate + StorageMutate diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index f398c20bab8..5189f3916c4 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -116,6 +116,7 @@ pub enum Column { DaCompressionTemporalRegistryPredicateCode = 22, /// Index of balances per user and asset. Balances = 23, + MessageBalances = 24, } impl Column { diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 876397f1f0c..50eb884178d 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -69,6 +69,25 @@ impl TableWithBlueprint for Balances { } } +/// These table stores the balances of asset id per owner. +pub struct MessageBalances; + +impl Mappable for MessageBalances { + type Key = Address; + type OwnedKey = Self::Key; + type Value = Amount; + type OwnedValue = Self::Value; +} + +impl TableWithBlueprint for MessageBalances { + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::MessageBalances + } +} + // TODO[RC]: These are most likely not needed, we're testing at integration level. // TODO[RC]: This needs to be additionally tested with a proper integration test // #[cfg(test)] diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 90d56b2a4f9..d1fb43e2759 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -20,7 +20,10 @@ use crate::{ worker::OffChainDatabaseTransaction, }, storage::{ - balances::Balances, + balances::{ + Balances, + MessageBalances, + }, blocks::FuelBlockIdsToHeights, coins::{ owner_coin_id_key, @@ -54,6 +57,7 @@ use fuel_core_storage::{ not_found, tables::ConsensusParametersVersions, Error as StorageError, + Mappable, Result as StorageResult, StorageAsMut, }; @@ -202,7 +206,8 @@ where } } -fn increase_balance( +// TODO[RC]: Maybe merge with `increase_message_balance()`? +fn increase_coin_balance( owner: &Address, asset_id: &AssetId, amount: Amount, @@ -211,15 +216,40 @@ fn increase_balance( where T: OffChainDatabaseTransaction, { + println!( + "increasing coin balance for owner: {:?}, asset_id: {:?}, amount: {:?}", + owner, asset_id, amount + ); + + // TODO[RC]: Make sure this operation is atomic + let key = BalancesKey::new(owner, asset_id); + let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); + let new_balance = current_balance.saturating_add(amount); + tx.storage_as_mut::().insert(&key, &new_balance) +} + +fn increase_message_balance( + owner: &Address, + amount: Amount, + tx: &mut T, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + println!( + "increasing message balance for owner: {:?}, amount: {:?}", + owner, amount + ); + // TODO[RC]: Make sure this operation is atomic - let balances_key = BalancesKey::new(owner, asset_id); + let key = owner; let current_balance = tx - .storage::() - .get(&balances_key)? + .storage::() + .get(&key)? .unwrap_or_default(); let new_balance = current_balance.saturating_add(amount); - tx.storage_as_mut::() - .insert(&balances_key, &new_balance) + tx.storage_as_mut::() + .insert(&key, &new_balance) } /// Process the executor events and update the indexes for the messages and coins. @@ -244,9 +274,8 @@ where )?; // *** "New" behavior (using Balances DB) *** - increase_balance( + increase_message_balance( &message.recipient(), - base_asset_id, message.amount(), block_st_transaction, )?; @@ -270,7 +299,7 @@ where .insert(&coin_by_owner, &())?; // *** "New" behavior (using Balances DB) *** - increase_balance( + increase_coin_balance( &coin.owner, &coin.asset_id, coin.amount, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 28e8e17db2e..135babca31e 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -20,6 +20,7 @@ use crate::{ balances::{ Balances, BalancesKey, + MessageBalances, }, old::{ OldFuelBlockConsensus, @@ -215,9 +216,14 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let base_asset_id = base_asset_id(); let messages = self - .storage_as_ref::() - .get(&BalancesKey::new(owner, &base_asset_id))? - .ok_or(not_found!(Balances))?; + .storage_as_ref::() + .get(&owner)? + .ok_or(not_found!(MessageBalances))?; + + println!( + "{coins} coins + {messages} messages = {}", + *coins + *messages + ); Ok(coins .checked_add(*messages) diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index 94d882f9fff..5079e83d444 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -235,14 +235,7 @@ async fn first_5_balances() { async fn foo_1() { let owner = Address::default(); - // TODO[RC]: This is just a hack - let asset_id = - Vec::from_hex("aabbccdd3d7fa5b672b530cbb84fcccb4ff8dc40f8176ef4544ddb1f1952ad07") - .unwrap(); - let arr: [u8; 32] = asset_id.try_into().unwrap(); - let asset_id = AssetId::new(arr); - - // let asset_id = AssetId::BASE; + let asset_id = AssetId::BASE; // setup config let mut coin_generator = CoinConfigGenerator::new(); @@ -337,3 +330,59 @@ async fn foo_1() { // let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); // assert_eq!(balance, 449); } + +#[tokio::test] +async fn foo_2() { + let owner = Address::default(); + let asset_id = AssetId::BASE; + + let different_asset_id = + Vec::from_hex("0606060606060606060606060606060606060606060606060606060606060606") + .unwrap(); + let arr: [u8; 32] = different_asset_id.try_into().unwrap(); + let different_asset_id = AssetId::new(arr); + + // setup config + let mut coin_generator = CoinConfigGenerator::new(); + let state_config = StateConfig { + contracts: vec![], + coins: vec![(owner, 1, asset_id), (owner, 10000, different_asset_id)] + .into_iter() + .map(|(owner, amount, asset_id)| CoinConfig { + owner, + amount, + asset_id, + ..coin_generator.generate() + }) + .collect(), + messages: vec![(owner, 2)] + .into_iter() + .enumerate() + .map(|(nonce, (owner, amount))| MessageConfig { + sender: owner, + recipient: owner, + nonce: (nonce as u64).into(), + amount, + data: vec![], + da_height: DaBlockHeight::from(0usize), + }) + .collect(), + ..Default::default() + }; + let config = Config::local_node_with_state_config(state_config); + + // setup server & client + let srv = FuelService::new_node(config).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + + // run test + let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); + assert_eq!(balance, 3); + + // run test + let balance = client + .balance(&owner, Some(&different_asset_id)) + .await + .unwrap(); + assert_eq!(balance, 10002); +} From 5595985a3a71dbd2c273375ba83d25149d5f439d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 15:01:54 +0200 Subject: [PATCH 025/111] Clean up column naming --- crates/fuel-core/src/graphql_api/storage.rs | 5 +++-- crates/fuel-core/src/graphql_api/storage/balances.rs | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index 5189f3916c4..de1db10a550 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -114,8 +114,9 @@ pub enum Column { DaCompressionTemporalRegistryScriptCode = 21, /// See [`DaCompressionTemporalRegistryPredicateCode`](da_compression::DaCompressionTemporalRegistryPredicateCode) DaCompressionTemporalRegistryPredicateCode = 22, - /// Index of balances per user and asset. - Balances = 23, + /// Coin balances per user and asset. + CoinBalances = 23, + /// Message balances per user. MessageBalances = 24, } diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 50eb884178d..a242a33d550 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -50,7 +50,7 @@ impl Distribution for Standard { } } -/// These table stores the balances of asset id per owner. +/// These table stores the balances of coins per owner and asset id. pub struct Balances; impl Mappable for Balances { @@ -65,11 +65,11 @@ impl TableWithBlueprint for Balances { type Column = super::Column; fn column() -> Self::Column { - Self::Column::Balances + Self::Column::CoinBalances } } -/// These table stores the balances of asset id per owner. +/// These table stores the balances of messages per owner. pub struct MessageBalances; impl Mappable for MessageBalances { From 38dd8d6eadb33dc12073fba2aba5e1da41bce1ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 16:49:14 +0200 Subject: [PATCH 026/111] Add test for coin balances --- .../src/graphql_api/worker_service.rs | 71 ++++++++- .../service/adapters/graphql_api/off_chain.rs | 4 +- tests/tests/balances.rs | 142 ++++++++++++++++-- 3 files changed, 197 insertions(+), 20 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index d1fb43e2759..25a842083a7 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -224,7 +224,32 @@ where // TODO[RC]: Make sure this operation is atomic let key = BalancesKey::new(owner, asset_id); let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); - let new_balance = current_balance.saturating_add(amount); + let new_balance = current_balance + .checked_add(amount) + .expect("coin balance too big"); + tx.storage_as_mut::().insert(&key, &new_balance) +} + +fn decrease_coin_balance( + owner: &Address, + asset_id: &AssetId, + amount: Amount, + tx: &mut T, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + println!( + "decreasing coin balance for owner: {:?}, asset_id: {:?}, amount: {:?}", + owner, asset_id, amount + ); + + // TODO[RC]: Make sure this operation is atomic + let key = BalancesKey::new(owner, asset_id); + let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); + let new_balance = current_balance + .checked_sub(amount) + .expect("can not spend more coin than a balance"); tx.storage_as_mut::().insert(&key, &new_balance) } @@ -247,7 +272,35 @@ where .storage::() .get(&key)? .unwrap_or_default(); - let new_balance = current_balance.saturating_add(amount); + let new_balance = current_balance + .checked_add(amount) + .expect("message balance too big"); + tx.storage_as_mut::() + .insert(&key, &new_balance) +} + +fn decrease_message_balance( + owner: &Address, + amount: Amount, + tx: &mut T, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + println!( + "increasing message balance for owner: {:?}, amount: {:?}", + owner, amount + ); + + // TODO[RC]: Make sure this operation is atomic + let key = owner; + let current_balance = tx + .storage::() + .get(&key)? + .unwrap_or_default(); + let new_balance = current_balance + .checked_sub(amount) + .expect("can not spend more messages than a balance"); tx.storage_as_mut::() .insert(&key, &new_balance) } @@ -281,6 +334,7 @@ where )?; } Event::MessageConsumed(message) => { + // *** "Old" behavior *** block_st_transaction .storage_as_mut::() .remove(&OwnedMessageKey::new( @@ -290,6 +344,13 @@ where block_st_transaction .storage::() .insert(message.nonce(), &())?; + + // *** "New" behavior (using Balances DB) *** + decrease_message_balance( + &message.recipient(), + message.amount(), + block_st_transaction, + )?; } Event::CoinCreated(coin) => { // *** "Old" behavior *** @@ -314,6 +375,12 @@ where .remove(&key)?; // *** "New" behavior (using Balances DB) *** + decrease_coin_balance( + &coin.owner, + &coin.asset_id, + coin.amount, + block_st_transaction, + )?; } Event::ForcedTransactionFailed { id, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 135babca31e..67b4189fcc3 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -211,14 +211,14 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let coins = self .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? - .ok_or(not_found!(Balances))?; + .unwrap_or_default(); let base_asset_id = base_asset_id(); let messages = self .storage_as_ref::() .get(&owner)? - .ok_or(not_found!(MessageBalances))?; + .unwrap_or_default(); println!( "{coins} coins + {messages} messages = {}", diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index 5079e83d444..4f763fd77f1 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -355,18 +355,18 @@ async fn foo_2() { ..coin_generator.generate() }) .collect(), - messages: vec![(owner, 2)] - .into_iter() - .enumerate() - .map(|(nonce, (owner, amount))| MessageConfig { - sender: owner, - recipient: owner, - nonce: (nonce as u64).into(), - amount, - data: vec![], - da_height: DaBlockHeight::from(0usize), - }) - .collect(), + // messages: vec![(owner, 2)] + // .into_iter() + // .enumerate() + // .map(|(nonce, (owner, amount))| MessageConfig { + // sender: owner, + // recipient: owner, + // nonce: (nonce as u64).into(), + // amount, + // data: vec![], + // da_height: DaBlockHeight::from(0usize), + // }) + // .collect(), ..Default::default() }; let config = Config::local_node_with_state_config(state_config); @@ -377,12 +377,122 @@ async fn foo_2() { // run test let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 3); - - // run test + assert_eq!(balance, 1); let balance = client .balance(&owner, Some(&different_asset_id)) .await .unwrap(); - assert_eq!(balance, 10002); + assert_eq!(balance, 10000); + + println!(); + + // spend COIN and check again + { + let coins_per_asset = client + .coins_to_spend(&owner, vec![(asset_id, 1, None)], None) + .await + .unwrap(); + dbg!(&coins_per_asset); + let mut tx = TransactionBuilder::script(vec![], vec![]) + .script_gas_limit(1_000_000) + .to_owned(); + for coins in coins_per_asset { + for coin in coins { + match coin { + CoinType::Coin(coin) => tx.add_input(Input::coin_signed( + coin.utxo_id, + coin.owner, + coin.amount, + coin.asset_id, + Default::default(), + 0, + )), + CoinType::MessageCoin(message) => { + tx.add_input(Input::message_coin_signed( + message.sender, + message.recipient, + message.amount, + message.nonce, + 0, + )) + } + CoinType::Unknown => panic!("Unknown coin"), + }; + } + } + + let tx = tx + .add_output(Output::Coin { + to: Address::new([1u8; 32]), + amount: 1, + asset_id, + }) + .add_witness(Default::default()) + .finalize_as_transaction(); + + client.submit_and_await_commit(&tx).await.unwrap(); + + let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); + assert_eq!(balance, 0); + } + + println!("FIRST CASE DONE"); + + // spend DIFFERENT COIN and check again + { + let coins_per_asset = client + .coins_to_spend(&owner, vec![(different_asset_id, 1, None)], None) + .await + .unwrap(); + dbg!(&coins_per_asset); + let mut tx = TransactionBuilder::script(vec![], vec![]) + .script_gas_limit(1_000_000) + .to_owned(); + for coins in coins_per_asset { + for coin in coins { + match coin { + CoinType::Coin(coin) => tx.add_input(Input::coin_signed( + coin.utxo_id, + coin.owner, + coin.amount, + coin.asset_id, + Default::default(), + 0, + )), + CoinType::MessageCoin(message) => { + tx.add_input(Input::message_coin_signed( + message.sender, + message.recipient, + message.amount, + message.nonce, + 0, + )) + } + CoinType::Unknown => panic!("Unknown coin"), + }; + } + } + + let tx = tx + .add_output(Output::Coin { + to: Address::new([2u8; 32]), + amount: 1, + asset_id: different_asset_id, + }) + .add_output(Output::Change { + to: owner, + amount: 9999, + asset_id: different_asset_id, + }) + .add_witness(Default::default()) + .finalize_as_transaction(); + + client.submit_and_await_commit(&tx).await.unwrap(); + + let balance = client + .balance(&owner, Some(&different_asset_id)) + .await + .unwrap(); + assert_eq!(balance, 9999); + } } From 530bcaf16883bdb0ca71e94eb9e4e2216f4e643d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 21 Oct 2024 17:42:53 +0200 Subject: [PATCH 027/111] Support both coins and messages in the new balance system --- crates/fuel-core/src/graphql_api/ports.rs | 12 +- .../src/graphql_api/worker_service.rs | 2 +- crates/fuel-core/src/query/balance.rs | 6 +- .../service/adapters/graphql_api/off_chain.rs | 37 +-- tests/tests/balances.rs | 218 ++++++++---------- tests/tests/lib.rs | 2 +- 6 files changed, 134 insertions(+), 143 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 98ce4e6a779..2404f79303c 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -71,7 +71,12 @@ pub trait OffChainDatabase: Send + Sync { fn tx_status(&self, tx_id: &TxId) -> StorageResult; - fn balance(&self, owner: &Address, asset_id: &AssetId) -> StorageResult; + fn balance( + &self, + owner: &Address, + asset_id: &AssetId, + base_asset_id: &AssetId, + ) -> StorageResult; fn owned_coins_ids( &self, @@ -275,7 +280,10 @@ pub mod worker { }, }, graphql_api::storage::{ - balances::{Balances, MessageBalances}, + balances::{ + Balances, + MessageBalances, + }, da_compression::*, old::{ OldFuelBlockConsensus, diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 25a842083a7..559e8997006 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -288,7 +288,7 @@ where T: OffChainDatabaseTransaction, { println!( - "increasing message balance for owner: {:?}, amount: {:?}", + "decreasing message balance for owner: {:?}, amount: {:?}", owner, amount ); diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 1b1d723eba7..02596ebb862 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -56,11 +56,13 @@ impl ReadView { // The new way. // TODO[RC]: balance could return both coins and messages - let amount = self.off_chain.balance(&owner, &asset_id)?; + let amount_1 = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; + + assert_eq!(amount, amount_1); Ok(AddressBalance { owner, - amount, + amount: amount_1, asset_id, }) } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 67b4189fcc3..2e5bde372ab 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -207,27 +207,38 @@ impl OffChainDatabase for OffChainIterableKeyValueView { self.message_is_spent(nonce) } - fn balance(&self, owner: &Address, asset_id: &AssetId) -> StorageResult { + fn balance( + &self, + owner: &Address, + asset_id: &AssetId, + base_asset_id: &AssetId, + ) -> StorageResult { let coins = self .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? .unwrap_or_default(); - let base_asset_id = base_asset_id(); + // let base_asset_id = base_asset_id(); - let messages = self - .storage_as_ref::() - .get(&owner)? - .unwrap_or_default(); + if base_asset_id == asset_id { + let messages = self + .storage_as_ref::() + .get(&owner)? + .unwrap_or_default(); + + println!( + "{coins} coins + {messages} messages = {}", + *coins + *messages + ); - println!( - "{coins} coins + {messages} messages = {}", - *coins + *messages - ); + Ok(coins + .checked_add(*messages) + .expect("TODO[RC]: balance too big")) + } else { + println!("{coins} coins"); - Ok(coins - .checked_add(*messages) - .expect("TODO[RC]: balance too big")) + Ok(*coins) + } } } diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index 4f763fd77f1..d8a6a36da14 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -231,106 +231,9 @@ async fn first_5_balances() { } } -#[tokio::test] -async fn foo_1() { - let owner = Address::default(); - - let asset_id = AssetId::BASE; - - // setup config - let mut coin_generator = CoinConfigGenerator::new(); - let state_config = StateConfig { - contracts: vec![], - coins: vec![ - (owner, 50, asset_id), - (owner, 100, asset_id), - (owner, 150, asset_id), - ] - .into_iter() - .map(|(owner, amount, asset_id)| CoinConfig { - owner, - amount, - asset_id, - ..coin_generator.generate() - }) - .collect(), - messages: vec![(owner, 60), (owner, 90)] - .into_iter() - .enumerate() - .map(|(nonce, (owner, amount))| MessageConfig { - sender: owner, - recipient: owner, - nonce: (nonce as u64).into(), - amount, - data: vec![], - da_height: DaBlockHeight::from(0usize), - }) - .collect(), - ..Default::default() - }; - let config = Config::local_node_with_state_config(state_config); - - // setup server & client - let srv = FuelService::new_node(config).await.unwrap(); - let client = FuelClient::from(srv.bound_address); - - // run test - let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 450); - - // spend some coins and check again - // let coins_per_asset = client - // .coins_to_spend(&owner, vec![(asset_id, 1, None)], None) - // .await - // .unwrap(); - - // let mut tx = TransactionBuilder::script(vec![], vec![]) - // .script_gas_limit(1_000_000) - // .to_owned(); - // for coins in coins_per_asset { - // for coin in coins { - // match coin { - // CoinType::Coin(coin) => tx.add_input(Input::coin_signed( - // coin.utxo_id, - // coin.owner, - // coin.amount, - // coin.asset_id, - // Default::default(), - // 0, - // )), - // CoinType::MessageCoin(message) => { - // tx.add_input(Input::message_coin_signed( - // message.sender, - // message.recipient, - // message.amount, - // message.nonce, - // 0, - // )) - // } - // CoinType::Unknown => panic!("Unknown coin"), - // }; - // } - // } - // let tx = tx - // .add_output(Output::Coin { - // to: Address::new([1u8; 32]), - // amount: 1, - // asset_id, - // }) - // .add_output(Output::Change { - // to: owner, - // amount: 0, - // asset_id, - // }) - // .add_witness(Default::default()) - // .finalize_as_transaction(); - - // client.submit_and_await_commit(&tx).await.unwrap(); - - // let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - // assert_eq!(balance, 449); -} - +// TODO[RC]: This is a temporary test that I used to debug the balance issue. +// I keep it here for now, but it should be removed when the balances caching is finished. +// Or make it more concise and use for thorough balance testing. #[tokio::test] async fn foo_2() { let owner = Address::default(); @@ -346,7 +249,7 @@ async fn foo_2() { let mut coin_generator = CoinConfigGenerator::new(); let state_config = StateConfig { contracts: vec![], - coins: vec![(owner, 1, asset_id), (owner, 10000, different_asset_id)] + coins: vec![(owner, 20, asset_id), (owner, 100, different_asset_id)] .into_iter() .map(|(owner, amount, asset_id)| CoinConfig { owner, @@ -355,18 +258,18 @@ async fn foo_2() { ..coin_generator.generate() }) .collect(), - // messages: vec![(owner, 2)] - // .into_iter() - // .enumerate() - // .map(|(nonce, (owner, amount))| MessageConfig { - // sender: owner, - // recipient: owner, - // nonce: (nonce as u64).into(), - // amount, - // data: vec![], - // da_height: DaBlockHeight::from(0usize), - // }) - // .collect(), + messages: vec![(owner, 10)] + .into_iter() + .enumerate() + .map(|(nonce, (owner, amount))| MessageConfig { + sender: owner, + recipient: owner, + nonce: (nonce as u64).into(), + amount, + data: vec![], + da_height: DaBlockHeight::from(0usize), + }) + .collect(), ..Default::default() }; let config = Config::local_node_with_state_config(state_config); @@ -377,14 +280,74 @@ async fn foo_2() { // run test let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 1); + assert_eq!(balance, 30); let balance = client .balance(&owner, Some(&different_asset_id)) .await .unwrap(); - assert_eq!(balance, 10000); + assert_eq!(balance, 100); - println!(); + println!( + "INIT PHASE COMPLETE, will be now sending 1 '0000...' coin to user '0101...'" + ); + + // spend DIFFERENT COIN and check again + { + let coins_per_asset = client + .coins_to_spend(&owner, vec![(different_asset_id, 1, None)], None) + .await + .unwrap(); + println!("coins to spend = {:#?}", &coins_per_asset); + let mut tx = TransactionBuilder::script(vec![], vec![]) + .script_gas_limit(1_000_000) + .to_owned(); + for coins in coins_per_asset { + for coin in coins { + match coin { + CoinType::Coin(coin) => tx.add_input(Input::coin_signed( + coin.utxo_id, + coin.owner, + coin.amount, + coin.asset_id, + Default::default(), + 0, + )), + CoinType::MessageCoin(message) => { + tx.add_input(Input::message_coin_signed( + message.sender, + message.recipient, + message.amount, + message.nonce, + 0, + )) + } + CoinType::Unknown => panic!("Unknown coin"), + }; + } + } + + let tx = tx + .add_output(Output::Coin { + to: Address::new([1u8; 32]), + amount: 2, + asset_id: different_asset_id, + }) + .add_output(Output::Change { + to: owner, + amount: 0, + asset_id: different_asset_id, + }) + .add_witness(Default::default()) + .finalize_as_transaction(); + + client.submit_and_await_commit(&tx).await.unwrap(); + + let balance = client + .balance(&owner, Some(&different_asset_id)) + .await + .unwrap(); + assert_eq!(balance, 98); + } // spend COIN and check again { @@ -392,7 +355,7 @@ async fn foo_2() { .coins_to_spend(&owner, vec![(asset_id, 1, None)], None) .await .unwrap(); - dbg!(&coins_per_asset); + println!("coins to spend = {:#?}", &coins_per_asset); let mut tx = TransactionBuilder::script(vec![], vec![]) .script_gas_limit(1_000_000) .to_owned(); @@ -427,16 +390,23 @@ async fn foo_2() { amount: 1, asset_id, }) + .add_output(Output::Change { + to: owner, + amount: 0, + asset_id, + }) .add_witness(Default::default()) .finalize_as_transaction(); client.submit_and_await_commit(&tx).await.unwrap(); let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 0); + assert_eq!(balance, 29); } - println!("FIRST CASE DONE"); + println!( + "INIT PHASE COMPLETE, will be now sending 2 '0606...' coin to user '0101...'" + ); // spend DIFFERENT COIN and check again { @@ -444,7 +414,7 @@ async fn foo_2() { .coins_to_spend(&owner, vec![(different_asset_id, 1, None)], None) .await .unwrap(); - dbg!(&coins_per_asset); + println!("coins to spend = {:#?}", &coins_per_asset); let mut tx = TransactionBuilder::script(vec![], vec![]) .script_gas_limit(1_000_000) .to_owned(); @@ -475,13 +445,13 @@ async fn foo_2() { let tx = tx .add_output(Output::Coin { - to: Address::new([2u8; 32]), - amount: 1, + to: Address::new([1u8; 32]), + amount: 3, asset_id: different_asset_id, }) .add_output(Output::Change { to: owner, - amount: 9999, + amount: 0, asset_id: different_asset_id, }) .add_witness(Default::default()) @@ -493,6 +463,6 @@ async fn foo_2() { .balance(&owner, Some(&different_asset_id)) .await .unwrap(); - assert_eq!(balance, 9999); + assert_eq!(balance, 95); } } diff --git a/tests/tests/lib.rs b/tests/tests/lib.rs index 5337e134358..3fdc70e3151 100644 --- a/tests/tests/lib.rs +++ b/tests/tests/lib.rs @@ -1,5 +1,5 @@ #![deny(unused_must_use)] -#![deny(warnings)] +#![allow(warnings)] // Tmp change mod balances; mod blob; From 562c087babc7589f7ba096d5a0ef3c9b80671c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 10:47:50 +0200 Subject: [PATCH 028/111] update comment --- crates/fuel-core/src/service/genesis/importer/off_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 54894d78708..7ec759b38a1 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -135,7 +135,7 @@ impl ImportTable for Handler { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here + let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here, but could be not needed after having separate DB for messages and coins worker_service::process_executor_events(events, tx, &base_asset_id)?; Ok(()) From 82f7a166784baad15872a324cbd1bc3cb01b8033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 11:08:53 +0200 Subject: [PATCH 029/111] Remove the database migration hack --- Cargo.lock | 27 ++++++++---- Cargo.toml | 13 +----- crates/fuel-core/src/combined_database.rs | 12 ------ crates/fuel-core/src/database.rs | 4 +- .../src/database/database_description.rs | 16 +------ crates/fuel-core/src/database/metadata.rs | 42 ------------------- crates/fuel-core/src/database/storage.rs | 26 ------------ crates/fuel-core/src/service.rs | 1 - 8 files changed, 21 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14d48bb073f..a2d7ba00012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,8 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3157,7 +3158,8 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3869,7 +3871,8 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", @@ -3901,7 +3904,8 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", @@ -3937,7 +3941,8 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" dependencies = [ "derive_more", "digest 0.10.7", @@ -3957,7 +3962,8 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" [[package]] name = "fuel-tx" @@ -3984,7 +3990,8 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4017,7 +4024,8 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4059,7 +4067,8 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" -source = "git+https://github.com/FuelLabs/fuel-vm.git?branch=1965_balances#8897a74693f93a621bcc0cf1288ca763d659056c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 1a76cc92ca0..877267d21b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,15 +142,4 @@ indicatif = { version = "0.17", default-features = false } itertools = { version = "0.12", default-features = false } insta = "1.8" tempfile = "3.4" -tikv-jemallocator = "0.5" - -[patch.crates-io] -fuel-vm-private = { git = 'https://github.com/FuelLabs/fuel-vm.git', package = "fuel-vm", branch = "1965_balances" } -#fuel-tx = { path = "../fuel-vm/fuel-tx" } -#fuel-asm = { path = "../fuel-vm/fuel-asm" } -#fuel-crypto = { path = "../fuel-vm/fuel-crypto" } -#fuel-derive = { path = "../fuel-vm/fuel-derive" } -#fuel-merkle = { path = "../fuel-vm/fuel-merkle" } -#fuel-storage = { path = "../fuel-vm/fuel-storage" } -#fuel-types = { path = "../fuel-vm/fuel-types" } -#fuel-vm = { path = "../fuel-vm/fuel-vm" } \ No newline at end of file +tikv-jemallocator = "0.5" \ No newline at end of file diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index b2e6c78c93e..f69649d2605 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -135,18 +135,6 @@ impl CombinedDatabase { ) } - pub fn migrate_metadata(&mut self) -> StorageResult<()> { - // Error: Off chain database is already initialized - // let mut unchecked_off_chain = - // self.off_chain().clone().into_genesis().map_err(|_| { - // anyhow::anyhow!("Off chain database is already initialized") - // })?; - // unchecked_off_chain.migrate_metadata()?; - - self.off_chain.migrate_metadata()?; - Ok(()) - } - pub fn check_version(&self) -> StorageResult<()> { self.on_chain.check_version()?; self.off_chain.check_version()?; diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 40ce7abb9c5..0e211725052 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -525,11 +525,9 @@ where .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::V2 { + &DatabaseMetadata::V1 { version: Description::version(), height: new_height, - // TODO[RC]: This value must NOT be updated here. - indexation_progress: Default::default(), }, )?; diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 74f49c12994..2849ea66d54 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -76,15 +76,7 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { /// The metadata of the database contains information about the version and its height. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { - V1 { - version: u32, - height: Height, - }, - V2 { - version: u32, - height: Height, - indexation_progress: HashMap, - }, + V1 { version: u32, height: Height }, } impl DatabaseMetadata { @@ -92,7 +84,6 @@ impl DatabaseMetadata { pub fn version(&self) -> u32 { match self { Self::V1 { version, .. } => *version, - Self::V2 { version, .. } => *version, } } @@ -100,7 +91,6 @@ impl DatabaseMetadata { pub fn height(&self) -> &Height { match self { Self::V1 { height, .. } => height, - Self::V2 { height, .. } => height, } } @@ -111,10 +101,6 @@ impl DatabaseMetadata { ) -> Option<&IndexationStatus> { match self { Self::V1 { height, .. } => None, - Self::V2 { - indexation_progress, - .. - } => indexation_progress.get(&indexation_type), } } } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 8dc6ee9dedb..a6da67c608e 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -25,7 +25,6 @@ use fuel_core_storage::{ StorageAsRef, StorageInspect, StorageMutate, - StorageMutateForced, }; use tracing::info; @@ -54,47 +53,6 @@ where } } -impl Database -where - Description: DatabaseDescription, - Self: StorageInspect, Error = StorageError> - + StorageMutateForced> - + Modifiable, -{ - // TODO[RC]: Add test covering this. - pub fn migrate_metadata(&mut self) -> StorageResult<()> { - let Some(current_metadata) = - self.storage::>().get(&())? - else { - return Ok(()); - }; - - dbg!(¤t_metadata); - - match current_metadata.as_ref() { - DatabaseMetadata::V1 { version, height } => { - let initial_progress = [ - (IndexationType::Balances, IndexationStatus::new()), - (IndexationType::CoinsToSpend, IndexationStatus::new()), - ]; - let new_metadata = DatabaseMetadata::V2 { - version: *version + 1, - height: *height, - indexation_progress: initial_progress.into_iter().collect(), - }; - info!("Migrating metadata from V1 to version V2..."); - dbg!(&new_metadata); - let x = self.storage_as_mut::>(); - x.replace_forced(&(), &new_metadata)?; - - info!("...Migrated!"); - Ok(()) - } - DatabaseMetadata::V2 { .. } => return Ok(()), - } - } -} - impl Database where Description: DatabaseDescription, diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index e1809b1f181..3ff88569b94 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -18,36 +18,10 @@ use fuel_core_storage::{ StorageInspect, StorageMut, StorageMutate, - StorageMutateForced, StorageWrite, }; use tracing::info; -impl StorageMutateForced for GenericDatabase -where - M: Mappable, - Self: Modifiable, - StructuredStorage: StorageInspect, - for<'a> StorageTransaction<&'a Storage>: StorageMutate, - GenericDatabase: ForcedCommitDatabase, -{ - fn replace_forced( - &mut self, - key: &::Key, - value: &::Value, - ) -> Result::OwnedValue>, Self::Error> { - let mut transaction = StorageTransaction::transaction( - self.as_ref(), - ConflictPolicy::Overwrite, - Default::default(), - ); - let prev = transaction.storage_as_mut::().replace(key, value)?; - let changes = transaction.into_changes(); - self.commit_changes_forced(changes)?; - Ok(prev) - } -} - impl StorageMutate for GenericDatabase where M: Mappable, diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index f6ec04f2644..6b42dd3960c 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -124,7 +124,6 @@ impl FuelService { // initialize state tracing::info!("Initializing database"); - database.migrate_metadata()?; database.check_version()?; Self::make_database_compatible_with_config( From 7d0b0148947efdbda56831f4009599e844a0950f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 11:20:49 +0200 Subject: [PATCH 030/111] Remove code related to handling base asset id --- Cargo.lock | 1 - crates/fuel-core/src/database.rs | 39 ------------------- .../src/database/database_description.rs | 16 -------- .../database_description/off_chain.rs | 4 +- crates/fuel-core/src/database/metadata.rs | 11 ------ crates/fuel-core/src/database/storage.rs | 8 +--- .../src/graphql_api/storage/balances.rs | 19 --------- .../src/graphql_api/worker_service.rs | 27 ------------- crates/fuel-core/src/lib.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 11 ------ .../service/genesis/importer/import_task.rs | 1 - .../src/service/genesis/importer/off_chain.rs | 13 ++----- crates/storage/Cargo.toml | 1 - crates/storage/src/lib.rs | 4 +- tests/tests/lib.rs | 2 +- 15 files changed, 9 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2d7ba00012..ee706d8560a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3682,7 +3682,6 @@ dependencies = [ "strum 0.25.0", "strum_macros 0.25.3", "test-case", - "tracing", ] [[package]] diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 0e211725052..d0b9dcdc21b 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -95,32 +95,6 @@ pub mod state; pub mod storage; pub mod transactions; -// TODO[RC]: Perhaps move to the new "indexation" module if indexation related structs grow too big. -#[derive( - Copy, Clone, Debug, serde::Serialize, serde::Deserialize, Hash, Eq, PartialEq, -)] -pub(crate) enum IndexationType { - Balances, - CoinsToSpend, -} - -#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] -pub(crate) enum IndexationStatus { - Pending, - CompletedUntil(BlockHeight), - Finished, -} - -impl IndexationStatus { - pub fn new() -> Self { - IndexationStatus::Pending - } - - pub fn is_finished(&self) -> bool { - matches!(self, IndexationStatus::Finished) - } -} - #[derive(Default, Debug, Copy, Clone)] pub struct GenesisStage; @@ -377,19 +351,6 @@ impl Modifiable for Database { } } -trait ForcedCommitDatabase { - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()>; -} - -impl ForcedCommitDatabase - for GenericDatabase>> -{ - fn commit_changes_forced(&mut self, changes: Changes) -> StorageResult<()> { - let mut height = *self.stage.height.lock(); - self.data.commit_changes(height, changes) - } -} - impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { commit_changes_with_height_update(self, changes, |iter| { diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 2849ea66d54..2bc2938f6fd 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -4,12 +4,6 @@ use fuel_core_types::{ blockchain::primitives::DaBlockHeight, fuel_types::BlockHeight, }; -use std::collections::HashMap; - -use super::{ - IndexationStatus, - IndexationType, -}; pub mod gas_price; pub mod off_chain; @@ -93,14 +87,4 @@ impl DatabaseMetadata { Self::V1 { height, .. } => height, } } - - /// Returns the indexation progress of a database - pub fn balances_indexation_progress( - &self, - indexation_type: IndexationType, - ) -> Option<&IndexationStatus> { - match self { - Self::V1 { height, .. } => None, - } - } } diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs index 9b0ce585901..1f339c50f3c 100644 --- a/crates/fuel-core/src/database/database_description/off_chain.rs +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -12,9 +12,7 @@ impl DatabaseDescription for OffChain { type Height = BlockHeight; fn version() -> u32 { - // TODO[RC]: Flip to 1, to take care of DatabaseMetadata::V2 - // TODO[RC]: This will fail the check_version(), do we need to migrate first? - 1 + 0 } fn name() -> String { diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index a6da67c608e..72cf2bbedb7 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -5,28 +5,17 @@ use crate::database::{ }, Database, Error as DatabaseError, - IndexationStatus, - IndexationType, }; use fuel_core_storage::{ blueprint::plain::Plain, codec::postcard::Postcard, structured_storage::TableWithBlueprint, - transactional::{ - Changes, - ConflictPolicy, - Modifiable, - StorageTransaction, - }, Error as StorageError, Mappable, Result as StorageResult, - StorageAsMut, StorageAsRef, StorageInspect, - StorageMutate, }; -use tracing::info; /// The table that stores all metadata about the database. pub struct MetadataTable(core::marker::PhantomData); diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 3ff88569b94..6222117aee3 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -1,11 +1,7 @@ -use crate::{ - database::ForcedCommitDatabase, - state::generic_database::GenericDatabase, -}; +use crate::state::generic_database::GenericDatabase; use fuel_core_storage::{ structured_storage::StructuredStorage, transactional::{ - Changes, ConflictPolicy, Modifiable, StorageTransaction, @@ -16,11 +12,9 @@ use fuel_core_storage::{ StorageAsMut, StorageBatchMutate, StorageInspect, - StorageMut, StorageMutate, StorageWrite, }; -use tracing::info; impl StorageMutate for GenericDatabase where diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index a242a33d550..a4b8823524f 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -1,18 +1,8 @@ -use fuel_core_chain_config::{ - AddTable, - AsTable, - StateConfig, - StateConfigBuilder, - TableEntry, -}; use fuel_core_storage::{ blueprint::plain::Plain, codec::{ - manual::Manual, postcard::Postcard, raw::Raw, - Decode, - Encode, }, structured_storage::TableWithBlueprint, Mappable, @@ -21,23 +11,14 @@ use fuel_core_types::{ fuel_tx::{ Address, AssetId, - Bytes32, - Bytes64, - Bytes8, }, - fuel_types::BlockHeight, fuel_vm::double_key, - services::txpool::TransactionStatus, }; use rand::{ distributions::Standard, prelude::Distribution, Rng, }; -use std::{ - array::TryFromSliceError, - mem::size_of, -}; pub type Amount = u64; diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 559e8997006..5112b60690e 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,5 +1,4 @@ use super::{ - api_service::ConsensusProvider, da_compression::da_compress_block, storage::{ balances::{ @@ -38,7 +37,6 @@ use crate::{ }, }, graphql_api::storage::relayed_transactions::RelayedTransactionStatuses, - service::adapters::ConsensusParametersProvider, }; use fuel_core_metrics::graphql_metrics::graphql_metrics; use fuel_core_services::{ @@ -50,14 +48,7 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ - iter::{ - IterDirection, - IteratorOverTable, - }, - not_found, - tables::ConsensusParametersVersions, Error as StorageError, - Mappable, Result as StorageResult, StorageAsMut, }; @@ -111,12 +102,10 @@ use futures::{ FutureExt, StreamExt, }; -use hex::FromHex; use std::{ borrow::Cow, ops::Deref, }; -use tracing::info; #[cfg(test)] mod tests; @@ -146,7 +135,6 @@ pub struct Task { block_importer: BoxStream, database: D, chain_id: ChainId, - base_asset_id: AssetId, da_compression_config: DaCompressionConfig, continue_on_error: bool, } @@ -181,7 +169,6 @@ where process_executor_events( result.events.iter().map(Cow::Borrowed), &mut transaction, - &self.base_asset_id, )?; match self.da_compression_config { @@ -309,7 +296,6 @@ where pub fn process_executor_events<'a, Iter, T>( events: Iter, block_st_transaction: &mut T, - base_asset_id: &AssetId, ) -> anyhow::Result<()> where Iter: Iterator>, @@ -603,16 +589,6 @@ where Ok(()) } -fn base_asset_id() -> AssetId { - // TODO[RC]: This is just a hack, get base asset id from consensus parameters here. - let base_asset_id = - Vec::from_hex("0000000000000000000000000000000000000000000000000000000000000000") - .unwrap(); - let arr: [u8; 32] = base_asset_id.try_into().unwrap(); - let base_asset_id = AssetId::new(arr); - base_asset_id -} - #[async_trait::async_trait] impl RunnableService for InitializeTask @@ -642,8 +618,6 @@ where graphql_metrics().total_txs_count.set(total_tx_count as i64); } - let base_asset_id = base_asset_id(); - let InitializeTask { chain_id, da_compression_config, @@ -662,7 +636,6 @@ where chain_id, da_compression_config, continue_on_error, - base_asset_id, }; let mut target_chain_height = on_chain_database.latest_height()?; diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index b30f960a022..40d866a137d 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![allow(warnings)] // tmp change to allow warnings +#![deny(warnings)] use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 2e5bde372ab..c3bca391b99 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -73,17 +73,6 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; -use hex::FromHex; - -fn base_asset_id() -> AssetId { - // TODO[RC]: This is just a hack, get base asset id from consensus parameters here. - let base_asset_id = - Vec::from_hex("0000000000000000000000000000000000000000000000000000000000000000") - .unwrap(); - let arr: [u8; 32] = base_asset_id.try_into().unwrap(); - let base_asset_id = AssetId::new(arr); - base_asset_id -} impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index bff148964d4..5cdcf636c8c 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -11,7 +11,6 @@ use fuel_core_storage::{ StorageInspect, StorageMutate, }; -use fuel_core_types::fuel_tx::AssetId; use crate::{ database::{ diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 7ec759b38a1..b5d96ddc603 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -35,10 +35,7 @@ use fuel_core_storage::{ transactional::StorageTransaction, StorageAsMut, }; -use fuel_core_types::{ - fuel_tx::AssetId, - services::executor::Event, -}; +use fuel_core_types::services::executor::Event; use std::borrow::Cow; use super::{ @@ -114,9 +111,7 @@ impl ImportTable for Handler { .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here - - worker_service::process_executor_events(events, tx, &base_asset_id)?; + worker_service::process_executor_events(events, tx)?; Ok(()) } } @@ -135,9 +130,7 @@ impl ImportTable for Handler { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - let base_asset_id = AssetId::default(); // TODO[RC]: Get base asset id here, but could be not needed after having separate DB for messages and coins - - worker_service::process_executor_events(events, tx, &base_asset_id)?; + worker_service::process_executor_events(events, tx)?; Ok(()) } } diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index af598c4c33d..8750f3ae8b0 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -17,7 +17,6 @@ repository = { workspace = true } version = { workspace = true } [dependencies] -tracing = { workspace = true } anyhow = { workspace = true } derive_more = { workspace = true } enum-iterator = { workspace = true } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index d7e89ca4ea1..c54fbf889b8 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -7,9 +7,9 @@ #![cfg_attr(not(feature = "std"), no_std)] #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] -#![allow(unused_crate_dependencies)] // tmp change to allow warnings +#![deny(unused_crate_dependencies)] #![deny(missing_docs)] -#![allow(warnings)] // tmp change to allow warnings +#![deny(warnings)] #[cfg(feature = "alloc")] extern crate alloc; diff --git a/tests/tests/lib.rs b/tests/tests/lib.rs index 3fdc70e3151..5337e134358 100644 --- a/tests/tests/lib.rs +++ b/tests/tests/lib.rs @@ -1,5 +1,5 @@ #![deny(unused_must_use)] -#![allow(warnings)] // Tmp change +#![deny(warnings)] mod balances; mod blob; From 1fe3bbad84d60933e050830f0c25d8969ec4707e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 12:37:52 +0200 Subject: [PATCH 031/111] Clean-up of the balance update code --- .../src/graphql_api/worker_service.rs | 113 +++++------------- crates/fuel-core/src/query/balance.rs | 1 - tests/tests/balances.rs | 2 +- 3 files changed, 30 insertions(+), 86 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 5112b60690e..c5e81a8de21 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -106,6 +106,7 @@ use std::{ borrow::Cow, ops::Deref, }; +use tracing::debug; #[cfg(test)] mod tests; @@ -193,101 +194,39 @@ where } } -// TODO[RC]: Maybe merge with `increase_message_balance()`? -fn increase_coin_balance( +fn update_coin_balance( owner: &Address, asset_id: &AssetId, amount: Amount, tx: &mut T, + updater: F, ) -> StorageResult<()> where T: OffChainDatabaseTransaction, + F: Fn(Cow, Amount) -> Amount, { - println!( - "increasing coin balance for owner: {:?}, asset_id: {:?}, amount: {:?}", - owner, asset_id, amount - ); - - // TODO[RC]: Make sure this operation is atomic let key = BalancesKey::new(owner, asset_id); let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); - let new_balance = current_balance - .checked_add(amount) - .expect("coin balance too big"); + let new_balance = updater(current_balance, amount); tx.storage_as_mut::().insert(&key, &new_balance) } -fn decrease_coin_balance( +fn update_message_balance( owner: &Address, - asset_id: &AssetId, amount: Amount, tx: &mut T, + updater: F, ) -> StorageResult<()> where T: OffChainDatabaseTransaction, + F: Fn(Cow, Amount) -> Amount, { - println!( - "decreasing coin balance for owner: {:?}, asset_id: {:?}, amount: {:?}", - owner, asset_id, amount - ); - - // TODO[RC]: Make sure this operation is atomic - let key = BalancesKey::new(owner, asset_id); - let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); - let new_balance = current_balance - .checked_sub(amount) - .expect("can not spend more coin than a balance"); - tx.storage_as_mut::().insert(&key, &new_balance) -} - -fn increase_message_balance( - owner: &Address, - amount: Amount, - tx: &mut T, -) -> StorageResult<()> -where - T: OffChainDatabaseTransaction, -{ - println!( - "increasing message balance for owner: {:?}, amount: {:?}", - owner, amount - ); - - // TODO[RC]: Make sure this operation is atomic - let key = owner; - let current_balance = tx - .storage::() - .get(&key)? - .unwrap_or_default(); - let new_balance = current_balance - .checked_add(amount) - .expect("message balance too big"); - tx.storage_as_mut::() - .insert(&key, &new_balance) -} - -fn decrease_message_balance( - owner: &Address, - amount: Amount, - tx: &mut T, -) -> StorageResult<()> -where - T: OffChainDatabaseTransaction, -{ - println!( - "decreasing message balance for owner: {:?}, amount: {:?}", - owner, amount - ); - - // TODO[RC]: Make sure this operation is atomic let key = owner; let current_balance = tx .storage::() .get(&key)? .unwrap_or_default(); - let new_balance = current_balance - .checked_sub(amount) - .expect("can not spend more messages than a balance"); + let new_balance = updater(current_balance, amount); tx.storage_as_mut::() .insert(&key, &new_balance) } @@ -304,7 +243,6 @@ where for event in events { match event.deref() { Event::MessageImported(message) => { - // *** "Old" behavior *** block_st_transaction .storage_as_mut::() .insert( @@ -312,15 +250,15 @@ where &(), )?; - // *** "New" behavior (using Balances DB) *** - increase_message_balance( - &message.recipient(), + debug!(recipient=%message.recipient(), amount=%message.amount(), "increasing message balance"); + update_message_balance( + message.recipient(), message.amount(), block_st_transaction, + |balance, amount| balance.saturating_add(amount), )?; } Event::MessageConsumed(message) => { - // *** "Old" behavior *** block_st_transaction .storage_as_mut::() .remove(&OwnedMessageKey::new( @@ -331,41 +269,48 @@ where .storage::() .insert(message.nonce(), &())?; - // *** "New" behavior (using Balances DB) *** - decrease_message_balance( - &message.recipient(), + debug!(recipient=%message.recipient(), amount=%message.amount(), "decreasing message balance"); + update_message_balance( + message.recipient(), message.amount(), block_st_transaction, + |balance, amount| balance.saturating_sub(amount), )?; } Event::CoinCreated(coin) => { - // *** "Old" behavior *** let coin_by_owner = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .insert(&coin_by_owner, &())?; - // *** "New" behavior (using Balances DB) *** - increase_coin_balance( + debug!( + owner=%coin.owner, + asset_id=%coin.asset_id, + amount=%coin.amount, "increasing coin balance"); + update_coin_balance( &coin.owner, &coin.asset_id, coin.amount, block_st_transaction, + |balance, amount| balance.saturating_add(amount), )?; } Event::CoinConsumed(coin) => { - // *** "Old" behavior *** let key = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .remove(&key)?; - // *** "New" behavior (using Balances DB) *** - decrease_coin_balance( + debug!( + owner=%coin.owner, + asset_id=%coin.asset_id, + amount=%coin.amount, "decreasing coin balance"); + update_coin_balance( &coin.owner, &coin.asset_id, coin.amount, block_st_transaction, + |balance, amount| balance.saturating_sub(amount), )?; } Event::ForcedTransactionFailed { diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 02596ebb862..a4455065031 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -55,7 +55,6 @@ impl ReadView { .await?; // The new way. - // TODO[RC]: balance could return both coins and messages let amount_1 = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; assert_eq!(amount, amount_1); diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index d8a6a36da14..4616584c674 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -17,7 +17,7 @@ use fuel_core_client::client::{ }, types::{ primitives::{ - Address, + Address, AssetId, }, CoinType, From fe60c98915082b08655a70fbba27dd82e047c92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 12:38:25 +0200 Subject: [PATCH 032/111] Remove unused tests --- .../src/graphql_api/storage/balances.rs | 249 ------------------ 1 file changed, 249 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index a4b8823524f..bf2723b9bcd 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -68,252 +68,3 @@ impl TableWithBlueprint for MessageBalances { Self::Column::MessageBalances } } - -// TODO[RC]: These are most likely not needed, we're testing at integration level. -// TODO[RC]: This needs to be additionally tested with a proper integration test -// #[cfg(test)] -// mod tests { -// use std::collections::HashMap; -// -// use fuel_core_storage::{ -// iter::IterDirection, -// StorageInspect, -// StorageMutate, -// }; -// use fuel_core_types::fuel_tx::{ -// Address, -// AssetId, -// Bytes64, -// Bytes8, -// }; -// -// use crate::{ -// combined_database::CombinedDatabase, -// graphql_api::storage::balances::Amount, -// }; -// -// use super::{ -// Balances, -// BalancesKey, -// }; -// -// pub struct TestDatabase { -// database: CombinedDatabase, -// } -// -// impl TestDatabase { -// pub fn new() -> Self { -// Self { -// database: Default::default(), -// } -// } -// -// pub fn register_amount( -// &mut self, -// owner: &Address, -// (asset_id, amount): &(AssetId, Amount), -// ) { -// let current_balance = self.query_balance(owner, asset_id); -// let new_balance = Amount { -// coins: current_balance.unwrap_or_default().coins + amount.coins, -// messages: current_balance.unwrap_or_default().messages + amount.messages, -// }; -// -// let db = self.database.off_chain_mut(); -// let key = BalancesKey::new(owner, asset_id); -// let _ = StorageMutate::::insert(db, &key, &new_balance) -// .expect("couldn't store test asset"); -// } -// -// pub fn query_balance( -// &self, -// owner: &Address, -// asset_id: &AssetId, -// ) -> Option { -// let db = self.database.off_chain(); -// let key = BalancesKey::new(owner, asset_id); -// let result = StorageInspect::::get(db, &key).unwrap(); -// -// result.map(|r| r.into_owned()) -// } -// -// pub fn query_balances(&self, owner: &Address) -> HashMap { -// let db = self.database.off_chain(); -// -// let mut key_prefix = owner.as_ref().to_vec(); -// db.entries::(Some(key_prefix), IterDirection::Forward) -// .map(|asset| { -// let asset = asset.expect("TODO[RC]: Fixme"); -// let asset_id = asset.key.asset_id().clone(); -// let balance = asset.value; -// (asset_id, balance) -// }) -// .collect() -// } -// } -// -// #[test] -// fn can_retrieve_balance_of_asset() { -// let mut db = TestDatabase::new(); -// -// let alice = Address::from([1; 32]); -// let bob = Address::from([2; 32]); -// let carol = Address::from([3; 32]); -// -// let ASSET_1 = AssetId::from([1; 32]); -// let ASSET_2 = AssetId::from([2; 32]); -// -// let alice_tx_1 = ( -// ASSET_1, -// Amount { -// coins: 100, -// messages: 0, -// }, -// ); -// let alice_tx_2 = ( -// ASSET_2, -// Amount { -// coins: 600, -// messages: 0, -// }, -// ); -// let alice_tx_3 = ( -// ASSET_2, -// Amount { -// coins: 400, -// messages: 0, -// }, -// ); -// -// Carol has 200 of asset 2 -// let carol_tx_1 = ( -// ASSET_2, -// Amount { -// coins: 200, -// messages: 0, -// }, -// ); -// -// let res = db.register_amount(&alice, &alice_tx_1); -// let res = db.register_amount(&alice, &alice_tx_2); -// let res = db.register_amount(&alice, &alice_tx_3); -// let res = db.register_amount(&carol, &carol_tx_1); -// -// Alice has correct balances -// assert_eq!( -// db.query_balance(&alice, &alice_tx_1.0), -// Some(Amount { -// coins: 100, -// messages: 0 -// }) -// ); -// assert_eq!( -// db.query_balance(&alice, &alice_tx_2.0), -// Some(Amount { -// coins: 1000, -// messages: 0 -// }) -// ); -// -// Carol has correct balances -// assert_eq!( -// db.query_balance(&carol, &carol_tx_1.0), -// Some(Amount { -// coins: 200, -// messages: 0 -// }) -// ); -// } -// -// #[test] -// fn can_retrieve_balances_of_all_assets_of_owner() { -// let mut db = TestDatabase::new(); -// -// let alice = Address::from([1; 32]); -// let bob = Address::from([2; 32]); -// let carol = Address::from([3; 32]); -// -// let ASSET_1 = AssetId::from([1; 32]); -// let ASSET_2 = AssetId::from([2; 32]); -// -// let alice_tx_1 = ( -// ASSET_1, -// Amount { -// coins: 100, -// messages: 0, -// }, -// ); -// let alice_tx_2 = ( -// ASSET_2, -// Amount { -// coins: 600, -// messages: 0, -// }, -// ); -// let alice_tx_3 = ( -// ASSET_2, -// Amount { -// coins: 400, -// messages: 0, -// }, -// ); -// -// let carol_tx_1 = ( -// ASSET_2, -// Amount { -// coins: 200, -// messages: 0, -// }, -// ); -// -// let res = db.register_amount(&alice, &alice_tx_1); -// let res = db.register_amount(&alice, &alice_tx_2); -// let res = db.register_amount(&alice, &alice_tx_3); -// let res = db.register_amount(&carol, &carol_tx_1); -// -// Verify Alice balances -// let expected: HashMap<_, _> = vec![ -// ( -// ASSET_1, -// Amount { -// coins: 100, -// messages: 0, -// }, -// ), -// ( -// ASSET_2, -// Amount { -// coins: 1000, -// messages: 0, -// }, -// ), -// ] -// .into_iter() -// .collect(); -// let actual = db.query_balances(&alice); -// assert_eq!(expected, actual); -// -// Verify Bob balances -// let actual = db.query_balances(&bob); -// assert_eq!(HashMap::new(), actual); -// -// Verify Carol balances -// let expected: HashMap<_, _> = vec![( -// ASSET_2, -// Amount { -// coins: 200, -// messages: 0, -// }, -// )] -// .into_iter() -// .collect(); -// let actual = db.query_balances(&carol); -// assert_eq!(expected, actual); -// } -// -// fuel_core_storage::basic_storage_tests!( -// Balances, -// ::Key::default(), -// ::Value::default() -// ); -// } From adef78e7933df0b20ce520afb6e0042a57fe67a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 12:52:37 +0200 Subject: [PATCH 033/111] Handle overflow in `balance` query --- .../service/adapters/graphql_api/off_chain.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index c3bca391b99..c73919ca635 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -73,6 +73,7 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; +use tracing::debug; impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { @@ -207,25 +208,20 @@ impl OffChainDatabase for OffChainIterableKeyValueView { .get(&BalancesKey::new(owner, asset_id))? .unwrap_or_default(); - // let base_asset_id = base_asset_id(); - if base_asset_id == asset_id { let messages = self .storage_as_ref::() .get(&owner)? .unwrap_or_default(); - println!( - "{coins} coins + {messages} messages = {}", - *coins + *messages - ); + let total = coins.checked_add(*messages).ok_or(anyhow::anyhow!( + "Total balance overflow: coins: {coins}, messages: {messages}" + ))?; - Ok(coins - .checked_add(*messages) - .expect("TODO[RC]: balance too big")) + debug!(%coins, %messages, total, "total balance"); + Ok(total) } else { - println!("{coins} coins"); - + debug!(%coins, "total balance"); Ok(*coins) } } From 9409d526a6bd866f02985b25eea8cb5fc3e9d58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 12:56:58 +0200 Subject: [PATCH 034/111] Clippy and formatting --- Cargo.toml | 2 +- crates/fuel-core/src/graphql_api/worker_service.rs | 4 ++-- .../fuel-core/src/service/adapters/graphql_api/off_chain.rs | 2 +- tests/tests/balances.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 877267d21b0..d80bf4f5bdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,4 +142,4 @@ indicatif = { version = "0.17", default-features = false } itertools = { version = "0.12", default-features = false } insta = "1.8" tempfile = "3.4" -tikv-jemallocator = "0.5" \ No newline at end of file +tikv-jemallocator = "0.5" diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index c5e81a8de21..51893b9d84c 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -224,11 +224,11 @@ where let key = owner; let current_balance = tx .storage::() - .get(&key)? + .get(key)? .unwrap_or_default(); let new_balance = updater(current_balance, amount); tx.storage_as_mut::() - .insert(&key, &new_balance) + .insert(key, &new_balance) } /// Process the executor events and update the indexes for the messages and coins. diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index c73919ca635..592469c5223 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -211,7 +211,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { if base_asset_id == asset_id { let messages = self .storage_as_ref::() - .get(&owner)? + .get(owner)? .unwrap_or_default(); let total = coins.checked_add(*messages).ok_or(anyhow::anyhow!( diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index 4616584c674..d8a6a36da14 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -17,7 +17,7 @@ use fuel_core_client::client::{ }, types::{ primitives::{ - Address, + Address, AssetId, }, CoinType, From 13b6db917077d0601eaca3ddd217fb0e83778ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 13:01:35 +0200 Subject: [PATCH 035/111] Update db tests after reverting changes to metadata --- crates/fuel-core/src/database.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index d0b9dcdc21b..d1d85dfe17e 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -61,7 +61,6 @@ use std::{ fmt::Debug, sync::Arc, }; -use tracing::info; pub use fuel_core_database::Error; pub type Result = core::result::Result; @@ -465,7 +464,6 @@ where (Some(prev_height), None) => { // In production, we shouldn't have cases where we call `commit_changes` with intermediate changes. // The commit always should contain all data for the corresponding height. - info!("XXXX - bailing here because new_height is not set"); return Err(DatabaseError::NewHeightIsNotSet { prev_height: prev_height.as_u64(), } @@ -910,10 +908,9 @@ mod tests { .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::::V2 { + &DatabaseMetadata::::V1 { version: Default::default(), height: Default::default(), - indexation_progress: Default::default(), }, ) .unwrap(); @@ -985,10 +982,9 @@ mod tests { // When let result = database.storage_as_mut::>().insert( &(), - &DatabaseMetadata::::V2 { + &DatabaseMetadata::::V1 { version: Default::default(), height: Default::default(), - indexation_progress: Default::default(), }, ); From 55f902515983ba05f07b95162665b4767d667d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 13:01:42 +0200 Subject: [PATCH 036/111] Add comment --- crates/fuel-core/src/query/balance.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index a4455065031..2e144858d82 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -57,6 +57,7 @@ impl ReadView { // The new way. let amount_1 = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; + // Safety check for the time of development. assert_eq!(amount, amount_1); Ok(AddressBalance { From d2158443e3b2a080b373c649ab5266a8e9cedcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 14:22:35 +0200 Subject: [PATCH 037/111] Add support for `balances()` (plural) query --- crates/fuel-core/src/graphql_api/ports.rs | 11 +- crates/fuel-core/src/query/balance.rs | 104 ++++++++++-------- .../service/adapters/graphql_api/off_chain.rs | 24 ++++ 3 files changed, 94 insertions(+), 45 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 2404f79303c..53424d66813 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -62,7 +62,10 @@ use fuel_core_types::{ }, tai64::Tai64, }; -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::Arc, +}; pub trait OffChainDatabase: Send + Sync { fn block_height(&self, block_id: &BlockId) -> StorageResult; @@ -78,6 +81,12 @@ pub trait OffChainDatabase: Send + Sync { base_asset_id: &AssetId, ) -> StorageResult; + fn balances( + &self, + owner: &Address, + base_asset_id: &AssetId, + ) -> StorageResult>; + fn owned_coins_ids( &self, owner: &Address, diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 2e144858d82..93187bb90d0 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -2,7 +2,6 @@ use crate::fuel_core_graphql_api::database::ReadView; use asset_query::{ AssetQuery, AssetSpendTarget, - AssetsQuery, }; use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ @@ -17,15 +16,12 @@ use fuel_core_types::{ services::graphql_api::AddressBalance, }; use futures::{ - FutureExt, + stream, Stream, StreamExt, TryStreamExt, }; -use std::{ - cmp::Ordering, - collections::HashMap, -}; +use tracing::debug; pub mod asset_query; @@ -70,50 +66,70 @@ impl ReadView { pub fn balances<'a>( &'a self, owner: &'a Address, - direction: IterDirection, + _direction: IterDirection, base_asset_id: &'a AssetId, ) -> impl Stream> + 'a { - let query = AssetsQuery::new(owner, None, None, self, base_asset_id); - let stream = query.coins(); + debug!("Querying balances for {:?}", owner); + let balances = self + .off_chain + .balances(owner, base_asset_id) + .expect("Fixme"); + + let ret: Vec = balances + .iter() + .map(|(asset_id, amount)| AddressBalance { + owner: *owner, + amount: *amount, + asset_id: *asset_id, + }) + .collect(); - stream - .try_fold( - HashMap::new(), - move |mut amounts_per_asset, coin| async move { - let amount: &mut u64 = amounts_per_asset - .entry(*coin.asset_id(base_asset_id)) - .or_default(); - *amount = amount.saturating_add(coin.amount()); - Ok(amounts_per_asset) - }, - ) + stream::iter(ret) + .map(Ok) .into_stream() - .try_filter_map(move |amounts_per_asset| async move { - let mut balances = amounts_per_asset - .into_iter() - .map(|(asset_id, amount)| AddressBalance { - owner: *owner, - amount, - asset_id, - }) - .collect::>(); + .yield_each(self.batch_size) - balances.sort_by(|l, r| { - if l.asset_id < r.asset_id { - Ordering::Less - } else { - Ordering::Greater - } - }); + // let query = AssetsQuery::new(owner, None, None, self, base_asset_id); + // let stream = query.coins(); - if direction == IterDirection::Reverse { - balances.reverse(); - } + // stream + // .try_fold( + // HashMap::new(), + // move |mut amounts_per_asset, coin| async move { + // let amount: &mut u64 = amounts_per_asset + // .entry(*coin.asset_id(base_asset_id)) + // .or_default(); + // *amount = amount.saturating_add(coin.amount()); + // Ok(amounts_per_asset) + // }, + // ) + // .into_stream() + // .try_filter_map(move |amounts_per_asset| async move { + // let mut balances = amounts_per_asset + // .into_iter() + // .map(|(asset_id, amount)| AddressBalance { + // owner: *owner, + // amount, + // asset_id, + // }) + // .collect::>(); - Ok(Some(futures::stream::iter(balances))) - }) - .map_ok(|stream| stream.map(Ok)) - .try_flatten() - .yield_each(self.batch_size) + // balances.sort_by(|l, r| { + // if l.asset_id < r.asset_id { + // Ordering::Less + // } else { + // Ordering::Greater + // } + // }); + + // if direction == IterDirection::Reverse { + // balances.reverse(); + // } + + // Ok(Some(futures::stream::iter(balances))) + // }) + // .map_ok(|stream| stream.map(Ok)) + // .try_flatten() + // .yield_each(self.batch_size) } } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 592469c5223..1d213845e78 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::{ database::{ database_description::off_chain::OffChain, @@ -225,6 +227,28 @@ impl OffChainDatabase for OffChainIterableKeyValueView { Ok(*coins) } } + + fn balances( + &self, + owner: &Address, + _base_asset_id: &AssetId, + ) -> StorageResult> { + // TODO[RC]: Use _base_asset_id to also iterate over 'MessageBalances'. + + let mut balances = HashMap::new(); + for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { + let key = balance_key?; + let asset_id = key.asset_id(); + let balance = self + .storage_as_ref::() + .get(&key)? + .unwrap_or_default(); + debug!(%owner, %asset_id, %balance, "balance entry"); + balances.insert(*asset_id, *balance); + } + + Ok(balances) + } } impl worker::OffChainDatabase for Database { From 53131a327efdbb1548272fc719d564bfc5f69f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 15:54:59 +0200 Subject: [PATCH 038/111] Handle error properly instead of using `expect()` --- crates/fuel-core/src/query/balance.rs | 39 ++++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 93187bb90d0..4b413bd2871 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -1,3 +1,5 @@ +use std::future; + use crate::fuel_core_graphql_api::database::ReadView; use asset_query::{ AssetQuery, @@ -70,24 +72,29 @@ impl ReadView { base_asset_id: &'a AssetId, ) -> impl Stream> + 'a { debug!("Querying balances for {:?}", owner); - let balances = self - .off_chain - .balances(owner, base_asset_id) - .expect("Fixme"); - let ret: Vec = balances - .iter() - .map(|(asset_id, amount)| AddressBalance { - owner: *owner, - amount: *amount, - asset_id: *asset_id, - }) - .collect(); + match self.off_chain.balances(owner, base_asset_id) { + Ok(balances) => { + stream::iter(balances.into_iter().map(|(asset_id, amount)| { + AddressBalance { + owner: *owner, + amount, + asset_id, + } + })) + .map(Ok) + .into_stream() + .yield_each(self.batch_size) + .left_stream() + } + Err(err) => stream::once(future::ready(Err(err))).right_stream(), + } - stream::iter(ret) - .map(Ok) - .into_stream() - .yield_each(self.batch_size) + // match self.off_chain.balances(owner, base_asset_id) { + // Ok(balances) => { + // } + // Err(err) => stream::once(future::ready(Err(err))).yield_each(self.batch_size), + // } // let query = AssetsQuery::new(owner, None, None, self, base_asset_id); // let stream = query.coins(); From 1e24aaad23abc39709ea79cd4d6275c46e248b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 16:23:11 +0200 Subject: [PATCH 039/111] Add support for `direction` in balances query --- crates/fuel-core/src/graphql_api/ports.rs | 4 +- crates/fuel-core/src/query/balance.rs | 67 +++---------------- .../service/adapters/graphql_api/off_chain.rs | 6 +- 3 files changed, 16 insertions(+), 61 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 53424d66813..0814407e28c 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -63,7 +63,7 @@ use fuel_core_types::{ tai64::Tai64, }; use std::{ - collections::HashMap, + collections::BTreeMap, sync::Arc, }; @@ -85,7 +85,7 @@ pub trait OffChainDatabase: Send + Sync { &self, owner: &Address, base_asset_id: &AssetId, - ) -> StorageResult>; + ) -> StorageResult>; fn owned_coins_ids( &self, diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 4b413bd2871..cdd867d7127 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -23,6 +23,7 @@ use futures::{ StreamExt, TryStreamExt, }; +use itertools::Either; use tracing::debug; pub mod asset_query; @@ -68,19 +69,22 @@ impl ReadView { pub fn balances<'a>( &'a self, owner: &'a Address, - _direction: IterDirection, + direction: IterDirection, base_asset_id: &'a AssetId, ) -> impl Stream> + 'a { debug!("Querying balances for {:?}", owner); match self.off_chain.balances(owner, base_asset_id) { Ok(balances) => { - stream::iter(balances.into_iter().map(|(asset_id, amount)| { - AddressBalance { - owner: *owner, - amount, - asset_id, - } + let iter = if direction == IterDirection::Reverse { + Either::Left(balances.into_iter().rev()) + } else { + Either::Right(balances.into_iter()) + }; + stream::iter(iter.map(|(asset_id, amount)| AddressBalance { + owner: *owner, + amount, + asset_id, })) .map(Ok) .into_stream() @@ -89,54 +93,5 @@ impl ReadView { } Err(err) => stream::once(future::ready(Err(err))).right_stream(), } - - // match self.off_chain.balances(owner, base_asset_id) { - // Ok(balances) => { - // } - // Err(err) => stream::once(future::ready(Err(err))).yield_each(self.batch_size), - // } - - // let query = AssetsQuery::new(owner, None, None, self, base_asset_id); - // let stream = query.coins(); - - // stream - // .try_fold( - // HashMap::new(), - // move |mut amounts_per_asset, coin| async move { - // let amount: &mut u64 = amounts_per_asset - // .entry(*coin.asset_id(base_asset_id)) - // .or_default(); - // *amount = amount.saturating_add(coin.amount()); - // Ok(amounts_per_asset) - // }, - // ) - // .into_stream() - // .try_filter_map(move |amounts_per_asset| async move { - // let mut balances = amounts_per_asset - // .into_iter() - // .map(|(asset_id, amount)| AddressBalance { - // owner: *owner, - // amount, - // asset_id, - // }) - // .collect::>(); - - // balances.sort_by(|l, r| { - // if l.asset_id < r.asset_id { - // Ordering::Less - // } else { - // Ordering::Greater - // } - // }); - - // if direction == IterDirection::Reverse { - // balances.reverse(); - // } - - // Ok(Some(futures::stream::iter(balances))) - // }) - // .map_ok(|stream| stream.map(Ok)) - // .try_flatten() - // .yield_each(self.batch_size) } } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 1d213845e78..6f52b998ea6 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; use crate::{ database::{ @@ -232,10 +232,10 @@ impl OffChainDatabase for OffChainIterableKeyValueView { &self, owner: &Address, _base_asset_id: &AssetId, - ) -> StorageResult> { + ) -> StorageResult> { // TODO[RC]: Use _base_asset_id to also iterate over 'MessageBalances'. - let mut balances = HashMap::new(); + let mut balances = BTreeMap::new(); for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { let key = balance_key?; let asset_id = key.asset_id(); From eeea199a1a6b5e3dcf4d27c913f80165cdd16f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 16:27:00 +0200 Subject: [PATCH 040/111] Remove safety check from the `balance` query --- crates/fuel-core/src/query/balance.rs | 30 ++------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index cdd867d7127..e92901d2121 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -1,10 +1,6 @@ use std::future; use crate::fuel_core_graphql_api::database::ReadView; -use asset_query::{ - AssetQuery, - AssetSpendTarget, -}; use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::IterDirection, @@ -35,33 +31,11 @@ impl ReadView { asset_id: AssetId, base_asset_id: AssetId, ) -> StorageResult { - // The old way. - let amount = AssetQuery::new( - &owner, - &AssetSpendTarget::new(asset_id, u64::MAX, u16::MAX), - &base_asset_id, - None, - self, - ) - .coins() - .map(|res| res.map(|coins| coins.amount())) - .try_fold(0u64, |balance, amount| { - async move { - // Increase the balance - Ok(balance.saturating_add(amount)) - } - }) - .await?; - - // The new way. - let amount_1 = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; - - // Safety check for the time of development. - assert_eq!(amount, amount_1); + let amount = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; Ok(AddressBalance { owner, - amount: amount_1, + amount, asset_id, }) } From e77f33b7c5f0fb34b732825847e219e4a1761f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 16:45:22 +0200 Subject: [PATCH 041/111] Fix balance logging --- crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 6f52b998ea6..6f8df4dfe78 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -223,7 +223,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { debug!(%coins, %messages, total, "total balance"); Ok(total) } else { - debug!(%coins, "total balance"); + debug!(%total, "total balance"); Ok(*coins) } } From c9b7e29a2ab1b930c697bd5799d11f33bce65170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 16:45:45 +0200 Subject: [PATCH 042/111] Include message balance in `balances()` query --- .../service/adapters/graphql_api/off_chain.rs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 6f8df4dfe78..933415e6682 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -231,20 +231,32 @@ impl OffChainDatabase for OffChainIterableKeyValueView { fn balances( &self, owner: &Address, - _base_asset_id: &AssetId, + base_asset_id: &AssetId, ) -> StorageResult> { - // TODO[RC]: Use _base_asset_id to also iterate over 'MessageBalances'. - let mut balances = BTreeMap::new(); for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { let key = balance_key?; let asset_id = key.asset_id(); - let balance = self + + let messages = if base_asset_id == asset_id { + *self + .storage_as_ref::() + .get(owner)? + .unwrap_or_default() + } else { + 0 + }; + + let coins = self .storage_as_ref::() .get(&key)? .unwrap_or_default(); - debug!(%owner, %asset_id, %balance, "balance entry"); - balances.insert(*asset_id, *balance); + + let total = coins.checked_add(messages).ok_or(anyhow::anyhow!( + "Total balance overflow: coins: {coins}, messages: {messages}" + ))?; + debug!(%owner, %asset_id, %total, "balance entry"); + balances.insert(*asset_id, total); } Ok(balances) From 076cb42f6b6d50a37023203a89b74570567f675b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 16:46:06 +0200 Subject: [PATCH 043/111] Revert "Fix balance logging" This reverts commit e77f33b7c5f0fb34b732825847e219e4a1761f07. --- crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 933415e6682..87323f6c678 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -223,7 +223,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { debug!(%coins, %messages, total, "total balance"); Ok(total) } else { - debug!(%total, "total balance"); + debug!(%coins, "total balance"); Ok(*coins) } } From f5e2a6d75bcc8e2a31ea49ba6c02246949ae2d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 17:10:50 +0200 Subject: [PATCH 044/111] Revert a couple of unintentional changes --- crates/fuel-core/src/database/database_description.rs | 2 +- crates/fuel-core/src/database/storage.rs | 3 +-- crates/fuel-core/src/service/genesis/importer/off_chain.rs | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 2bc2938f6fd..14d240c54f5 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -68,7 +68,7 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { } /// The metadata of the database contains information about the version and its height. -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { V1 { version: u32, height: Height }, } diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 6222117aee3..d00e292963f 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -34,8 +34,7 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().replace(key, value)?; - let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index b5d96ddc603..eef13bf9ee5 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -110,7 +110,6 @@ impl ImportTable for Handler { let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - worker_service::process_executor_events(events, tx)?; Ok(()) } @@ -129,7 +128,6 @@ impl ImportTable for Handler { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - worker_service::process_executor_events(events, tx)?; Ok(()) } From d1c6fcc207505d1ef2a2109775178f8d179c3aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 17:20:41 +0200 Subject: [PATCH 045/111] Rename database `Balances` to `CoinBalances` --- crates/fuel-core/src/graphql_api/ports.rs | 4 ++-- crates/fuel-core/src/graphql_api/storage/balances.rs | 6 +++--- crates/fuel-core/src/graphql_api/worker_service.rs | 6 +++--- .../src/service/adapters/graphql_api/off_chain.rs | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 0814407e28c..225d93509b6 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -290,7 +290,7 @@ pub mod worker { }, graphql_api::storage::{ balances::{ - Balances, + CoinBalances, MessageBalances, }, da_compression::*, @@ -347,7 +347,7 @@ pub mod worker { + StorageMutate + StorageMutate + StorageMutate - + StorageMutate + + StorageMutate + StorageMutate + StorageMutate + StorageMutate diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index bf2723b9bcd..b70c80b9280 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -32,16 +32,16 @@ impl Distribution for Standard { } /// These table stores the balances of coins per owner and asset id. -pub struct Balances; +pub struct CoinBalances; -impl Mappable for Balances { +impl Mappable for CoinBalances { type Key = BalancesKey; type OwnedKey = Self::Key; type Value = Amount; type OwnedValue = Self::Value; } -impl TableWithBlueprint for Balances { +impl TableWithBlueprint for CoinBalances { type Blueprint = Plain; // TODO[RC]: What is Plain, Raw, Postcard, Primitive and others in this context? type Column = super::Column; diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 51893b9d84c..a4e563bdbd4 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -20,7 +20,7 @@ use crate::{ }, storage::{ balances::{ - Balances, + CoinBalances, MessageBalances, }, blocks::FuelBlockIdsToHeights, @@ -206,9 +206,9 @@ where F: Fn(Cow, Amount) -> Amount, { let key = BalancesKey::new(owner, asset_id); - let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); + let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); let new_balance = updater(current_balance, amount); - tx.storage_as_mut::().insert(&key, &new_balance) + tx.storage_as_mut::().insert(&key, &new_balance) } fn update_message_balance( diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 87323f6c678..1c201691234 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -20,7 +20,7 @@ use crate::{ }, graphql_api::storage::{ balances::{ - Balances, + CoinBalances, BalancesKey, MessageBalances, }, @@ -206,7 +206,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { base_asset_id: &AssetId, ) -> StorageResult { let coins = self - .storage_as_ref::() + .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? .unwrap_or_default(); @@ -234,7 +234,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { base_asset_id: &AssetId, ) -> StorageResult> { let mut balances = BTreeMap::new(); - for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { + for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { let key = balance_key?; let asset_id = key.asset_id(); @@ -248,7 +248,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { }; let coins = self - .storage_as_ref::() + .storage_as_ref::() .get(&key)? .unwrap_or_default(); From 8cc02802b6cad7d9b8ff0d20116fd555699e82bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 22 Oct 2024 17:51:41 +0200 Subject: [PATCH 046/111] Update comments --- Cargo.lock | 22 ++++++++++--------- .../src/graphql_api/storage/balances.rs | 6 ++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee706d8560a..d4d274e7f71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,7 +735,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.11.0", + "uuid 1.10.0", ] [[package]] @@ -2246,7 +2246,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid 1.11.0", + "uuid 1.10.0", ] [[package]] @@ -3226,7 +3226,7 @@ dependencies = [ "tower", "tower-http 0.4.4", "tracing", - "uuid 1.11.0", + "uuid 1.10.0", ] [[package]] @@ -3558,6 +3558,8 @@ dependencies = [ "serde", "serde_with", "sha2 0.10.8", + "strum 0.25.0", + "strum_macros 0.25.3", "thiserror", "tokio", "tracing", @@ -4726,9 +4728,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -7846,9 +7848,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -8628,7 +8630,7 @@ dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid 1.11.0", + "uuid 1.10.0", ] [[package]] @@ -9486,9 +9488,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.11.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", ] diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index b70c80b9280..20dd21de3b8 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -31,7 +31,7 @@ impl Distribution for Standard { } } -/// These table stores the balances of coins per owner and asset id. +/// This table stores the balances of coins per owner and asset id. pub struct CoinBalances; impl Mappable for CoinBalances { @@ -42,7 +42,7 @@ impl Mappable for CoinBalances { } impl TableWithBlueprint for CoinBalances { - type Blueprint = Plain; // TODO[RC]: What is Plain, Raw, Postcard, Primitive and others in this context? + type Blueprint = Plain; type Column = super::Column; fn column() -> Self::Column { @@ -50,7 +50,7 @@ impl TableWithBlueprint for CoinBalances { } } -/// These table stores the balances of messages per owner. +/// This table stores the balances of messages per owner. pub struct MessageBalances; impl Mappable for MessageBalances { From f08229158ba3dc80153eeb9c9d44eadbacf5c4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 23 Oct 2024 11:37:22 +0200 Subject: [PATCH 047/111] Fix formatting --- crates/fuel-core/src/graphql_api/worker_service.rs | 3 ++- crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index a4e563bdbd4..527dc2f9721 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -208,7 +208,8 @@ where let key = BalancesKey::new(owner, asset_id); let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); let new_balance = updater(current_balance, amount); - tx.storage_as_mut::().insert(&key, &new_balance) + tx.storage_as_mut::() + .insert(&key, &new_balance) } fn update_message_balance( diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 1c201691234..f9e6df54e90 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -20,8 +20,8 @@ use crate::{ }, graphql_api::storage::{ balances::{ - CoinBalances, BalancesKey, + CoinBalances, MessageBalances, }, old::{ From 8ce455036d5eda480832654ebbc4c6686c491c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 24 Oct 2024 16:18:33 +0200 Subject: [PATCH 048/111] Add comment about potential discount on the balances query cost --- crates/fuel-core/src/graphql_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 772bbc815ea..24e70c1dcbf 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -79,7 +79,7 @@ impl Default for Costs { } pub const DEFAULT_QUERY_COSTS: Costs = Costs { - balance_query: 40001, + balance_query: 40001, /* TODO[RC]: We might consider making this query cheaper because it's just a single DB read now. */ coins_to_spend: 40001, get_peers: 40001, estimate_predicates: 40001, From 4e84ac3ff5f26339e690743d6040eed56eb72728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 11:23:31 +0100 Subject: [PATCH 049/111] Update DB metadata with balances info --- crates/fuel-core/src/database.rs | 23 +++- .../src/database/database_description.rs | 38 ++++++- .../database_description/off_chain.rs | 1 + crates/fuel-core/src/database/metadata.rs | 10 ++ crates/fuel-core/src/graphql_api/database.rs | 3 + crates/fuel-core/src/graphql_api/ports.rs | 8 ++ .../src/graphql_api/worker_service.rs | 105 ++++++++++++------ crates/fuel-core/src/lib.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 16 ++- crates/fuel-core/src/service/genesis.rs | 20 ++++ .../src/service/genesis/importer/off_chain.rs | 5 +- 11 files changed, 186 insertions(+), 45 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index d1d85dfe17e..77cf95fb868 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -24,6 +24,7 @@ use crate::{ KeyValueView, }, }; +use database_description::IndexationKind; use fuel_core_chain_config::TableEntry; use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; use fuel_core_services::SharedMutex; @@ -58,6 +59,7 @@ use fuel_core_types::{ }; use itertools::Itertools; use std::{ + collections::HashSet, fmt::Debug, sync::Arc, }; @@ -480,13 +482,32 @@ where ConflictPolicy::Overwrite, changes, ); + + // TODO[RC]: Problems with this code: + // 1. additional DB read (of prev metadata) + // 2. HashSet collect for Metadata V2 + let current_metadata = transaction + .storage::>() + .get(&()) + .map_err(StorageError::from)? + .unwrap(); + let indexation_availability = match current_metadata.as_ref() { + DatabaseMetadata::V1 { version, height } => HashSet::new(), + DatabaseMetadata::V2 { + version, + height, + indexation_availability, + } => [(IndexationKind::Balances)].into_iter().collect(), + }; + transaction .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::V1 { + &DatabaseMetadata::V2 { version: Description::version(), height: new_height, + indexation_availability, }, )?; diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 14d240c54f5..9fe214d1478 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -4,6 +4,11 @@ use fuel_core_types::{ blockchain::primitives::DaBlockHeight, fuel_types::BlockHeight, }; +use off_chain::OffChain; +use std::collections::{ + HashMap, + HashSet, +}; pub mod gas_price; pub mod off_chain; @@ -67,10 +72,26 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { fn prefix(column: &Self::Column) -> Option; } +#[derive( + Copy, Clone, Debug, serde::Serialize, serde::Deserialize, Eq, PartialEq, Hash, +)] +pub enum IndexationKind { + Balances, + CoinsToSpend, +} + /// The metadata of the database contains information about the version and its height. -#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { - V1 { version: u32, height: Height }, + V1 { + version: u32, + height: Height, + }, + V2 { + version: u32, + height: Height, + indexation_availability: HashSet, + }, } impl DatabaseMetadata { @@ -78,6 +99,7 @@ impl DatabaseMetadata { pub fn version(&self) -> u32 { match self { Self::V1 { version, .. } => *version, + Self::V2 { version, .. } => *version, } } @@ -85,6 +107,18 @@ impl DatabaseMetadata { pub fn height(&self) -> &Height { match self { Self::V1 { height, .. } => height, + Self::V2 { height, .. } => height, + } + } + + /// Returns true if the given indexation kind is available. + pub fn indexation_available(&self, kind: IndexationKind) -> bool { + match self { + Self::V1 { .. } => false, + Self::V2 { + indexation_availability, + .. + } => indexation_availability.contains(&kind), } } } diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs index 1f339c50f3c..a29becf2d2a 100644 --- a/crates/fuel-core/src/database/database_description/off_chain.rs +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -11,6 +11,7 @@ impl DatabaseDescription for OffChain { type Column = fuel_core_graphql_api::storage::Column; type Height = BlockHeight; + // TODO[RC]: Do we bump this due to extended metadata? fn version() -> u32 { 0 } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 72cf2bbedb7..e538a863be5 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::database::{ database_description::{ DatabaseDescription, @@ -15,6 +17,7 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, StorageInspect, + StorageMutate, }; /// The table that stores all metadata about the database. @@ -74,4 +77,11 @@ where Ok(metadata) } + + // TODO[RC]: Needed? + pub fn metadata( + &self, + ) -> StorageResult>>> { + self.storage::>().get(&()) + } } diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index bf47c8d92a7..10bdf4d032e 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -86,6 +86,8 @@ pub struct ReadDatabase { on_chain: Box>, /// The off-chain database view provider. off_chain: Box>, + /// The flag that indicates whether the Balances cache table is enabled. + balances_enabled: bool, } impl ReadDatabase { @@ -107,6 +109,7 @@ impl ReadDatabase { genesis_height, on_chain: Box::new(ArcWrapper::new(on_chain)), off_chain: Box::new(ArcWrapper::new(off_chain)), + balances_enabled: false, // TODO[RC]: Read this properly } } diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 225d93509b6..27c509e3d50 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -280,6 +280,10 @@ pub trait MemoryPool { pub mod worker { use super::super::storage::blocks::FuelBlockIdsToHeights; use crate::{ + database::{ + database_description::off_chain::OffChain, + metadata::MetadataTable, + }, fuel_core_graphql_api::storage::{ coins::OwnedCoins, contracts::ContractsInfo, @@ -306,6 +310,7 @@ pub mod worker { use fuel_core_storage::{ Error as StorageError, Result as StorageResult, + StorageInspect, StorageMutate, }; use fuel_core_types::{ @@ -335,6 +340,9 @@ pub mod worker { /// Creates a write database transaction. fn transaction(&mut self) -> Self::Transaction<'_>; + + /// Checks if Balances cache table is available. + fn balances_enabled(&self) -> bool; } pub trait OffChainDatabaseTransaction: diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 527dc2f9721..400fa819220 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -13,6 +13,10 @@ use super::{ }, }; use crate::{ + database::{ + database_description::off_chain::OffChain, + metadata::MetadataTable, + }, fuel_core_graphql_api::{ ports::{ self, @@ -106,7 +110,11 @@ use std::{ borrow::Cow, ops::Deref, }; -use tracing::debug; +use tracing::{ + debug, + error, + info, +}; #[cfg(test)] mod tests; @@ -138,6 +146,7 @@ pub struct Task { chain_id: ChainId, da_compression_config: DaCompressionConfig, continue_on_error: bool, + balances_enabled: bool, } impl Task @@ -170,6 +179,7 @@ where process_executor_events( result.events.iter().map(Cow::Borrowed), &mut transaction, + self.balances_enabled, )?; match self.da_compression_config { @@ -207,7 +217,16 @@ where { let key = BalancesKey::new(owner, asset_id); let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); + let prev_balance = current_balance.clone(); let new_balance = updater(current_balance, amount); + debug!( + %owner, + %asset_id, + amount, + %prev_balance, + new_balance, + "changing coin balance" + ); tx.storage_as_mut::() .insert(&key, &new_balance) } @@ -227,7 +246,14 @@ where .storage::() .get(key)? .unwrap_or_default(); + let prev_balance = current_balance.clone(); let new_balance = updater(current_balance, amount); + debug!( + %owner, + %amount, + %prev_balance, + new_balance, + "changing message balance"); tx.storage_as_mut::() .insert(key, &new_balance) } @@ -236,6 +262,7 @@ where pub fn process_executor_events<'a, Iter, T>( events: Iter, block_st_transaction: &mut T, + balances_enabled: bool, ) -> anyhow::Result<()> where Iter: Iterator>, @@ -251,13 +278,15 @@ where &(), )?; - debug!(recipient=%message.recipient(), amount=%message.amount(), "increasing message balance"); - update_message_balance( - message.recipient(), - message.amount(), - block_st_transaction, - |balance, amount| balance.saturating_add(amount), - )?; + // TODO[RC]: Refactor to have this if called only once + if balances_enabled { + update_message_balance( + message.recipient(), + message.amount(), + block_st_transaction, + |balance, amount| balance.saturating_add(amount), + )?; + } } Event::MessageConsumed(message) => { block_st_transaction @@ -270,13 +299,15 @@ where .storage::() .insert(message.nonce(), &())?; - debug!(recipient=%message.recipient(), amount=%message.amount(), "decreasing message balance"); - update_message_balance( - message.recipient(), - message.amount(), - block_st_transaction, - |balance, amount| balance.saturating_sub(amount), - )?; + // TODO[RC]: Check other places where we update "OwnedCoins" or "OwnedMessageIds" + if balances_enabled { + update_message_balance( + message.recipient(), + message.amount(), + block_st_transaction, + |balance, amount| balance.saturating_sub(amount), + )?; + } } Event::CoinCreated(coin) => { let coin_by_owner = owner_coin_id_key(&coin.owner, &coin.utxo_id); @@ -284,17 +315,15 @@ where .storage_as_mut::() .insert(&coin_by_owner, &())?; - debug!( - owner=%coin.owner, - asset_id=%coin.asset_id, - amount=%coin.amount, "increasing coin balance"); - update_coin_balance( - &coin.owner, - &coin.asset_id, - coin.amount, - block_st_transaction, - |balance, amount| balance.saturating_add(amount), - )?; + if balances_enabled { + update_coin_balance( + &coin.owner, + &coin.asset_id, + coin.amount, + block_st_transaction, + |balance, amount| balance.saturating_add(amount), + )?; + } } Event::CoinConsumed(coin) => { let key = owner_coin_id_key(&coin.owner, &coin.utxo_id); @@ -302,17 +331,15 @@ where .storage_as_mut::() .remove(&key)?; - debug!( - owner=%coin.owner, - asset_id=%coin.asset_id, - amount=%coin.amount, "decreasing coin balance"); - update_coin_balance( - &coin.owner, - &coin.asset_id, - coin.amount, - block_st_transaction, - |balance, amount| balance.saturating_sub(amount), - )?; + if balances_enabled { + update_coin_balance( + &coin.owner, + &coin.asset_id, + coin.amount, + block_st_transaction, + |balance, amount| balance.saturating_sub(amount), + )?; + } } Event::ForcedTransactionFailed { id, @@ -564,6 +591,9 @@ where graphql_metrics().total_txs_count.set(total_tx_count as i64); } + let balances_enabled = self.off_chain_database.balances_enabled(); + info!("Balances cache available: {}", balances_enabled); + let InitializeTask { chain_id, da_compression_config, @@ -582,6 +612,7 @@ where chain_id, da_compression_config, continue_on_error, + balances_enabled, }; let mut target_chain_height = on_chain_database.latest_height()?; diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index 40d866a137d..8abd34e1f3d 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![deny(warnings)] +#![allow(warnings)] use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index f9e6df54e90..677247bee30 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -2,7 +2,10 @@ use std::collections::BTreeMap; use crate::{ database::{ - database_description::off_chain::OffChain, + database_description::{ + off_chain::OffChain, + IndexationKind, + }, Database, OffChainIterableKeyValueView, }, @@ -75,7 +78,10 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; -use tracing::debug; +use tracing::{ + debug, + error, +}; impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { @@ -273,4 +279,10 @@ impl worker::OffChainDatabase for Database { fn transaction(&mut self) -> Self::Transaction<'_> { self.into_transaction() } + + fn balances_enabled(&self) -> bool { + let metadata = self.metadata().unwrap().unwrap(); // TODO[RC]: Clean-up + error!(?metadata, "METADATA"); + metadata.indexation_available(IndexationKind::Balances) + } } diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index 55d36e2821b..27a74473854 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use self::importer::SnapshotImporter; use crate::{ combined_database::{ @@ -8,12 +10,17 @@ use crate::{ database_description::{ off_chain::OffChain, on_chain::OnChain, + DatabaseDescription, + DatabaseMetadata, + IndexationKind, }, genesis_progress::GenesisMetadata, + metadata::MetadataTable, Database, }, service::config::Config, }; +use async_graphql::Description; use fuel_core_chain_config::GenesisCommitment; use fuel_core_services::StateWatcher; use fuel_core_storage::{ @@ -130,6 +137,19 @@ pub async fn execute_genesis_block( .storage_as_mut::>() .remove(&key)?; } + + database_transaction_off_chain + .storage_as_mut::>() + .insert( + &(), + &DatabaseMetadata::V2 { + version: ::version(), + height: Default::default(), + indexation_availability: [(IndexationKind::Balances)] + .into_iter() + .collect(), + }, + )?; database_transaction_off_chain.commit()?; let mut database_transaction_on_chain = db.on_chain().read_transaction(); diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index eef13bf9ee5..cedcfbf6e41 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -1,6 +1,7 @@ use crate::{ database::{ database_description::off_chain::OffChain, + metadata::MetadataTable, GenesisDatabase, }, fuel_core_graphql_api::storage::messages::SpentMessages, @@ -110,7 +111,7 @@ impl ImportTable for Handler { let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - worker_service::process_executor_events(events, tx)?; + worker_service::process_executor_events(events, tx, true)?; Ok(()) } } @@ -128,7 +129,7 @@ impl ImportTable for Handler { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - worker_service::process_executor_events(events, tx)?; + worker_service::process_executor_events(events, tx, true)?; Ok(()) } } From 9577f303a93c792327918c92ca33698ae99fad44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 12:10:31 +0100 Subject: [PATCH 050/111] Support both 'new' and 'old' way of querying balances --- .../fuel-core/src/graphql_api/api_service.rs | 3 +- crates/fuel-core/src/graphql_api/database.rs | 26 ++++-- crates/fuel-core/src/query/balance.rs | 86 +++++++++++++++++-- 3 files changed, 102 insertions(+), 13 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index 28a714f8b0e..dec8883a503 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -90,6 +90,7 @@ use tower_http::{ pub type Service = fuel_core_services::ServiceRunner; pub use super::database::ReadDatabase; +use super::ports::worker; pub type BlockProducer = Box; // In the future GraphQL should not be aware of `TxPool`. It should @@ -229,7 +230,7 @@ pub fn new_service( ) -> anyhow::Result where OnChain: AtomicView + 'static, - OffChain: AtomicView + 'static, + OffChain: AtomicView + worker::OffChainDatabase + 'static, OnChain::LatestView: OnChainDatabase, OffChain::LatestView: OffChainDatabase, { diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 10bdf4d032e..b7494b888b6 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -1,8 +1,11 @@ -use crate::fuel_core_graphql_api::{ - database::arc_wrapper::ArcWrapper, - ports::{ - OffChainDatabase, - OnChainDatabase, +use crate::{ + database::database_description::DatabaseDescription, + fuel_core_graphql_api::{ + database::arc_wrapper::ArcWrapper, + ports::{ + OffChainDatabase, + OnChainDatabase, + }, }, }; use fuel_core_services::yield_stream::StreamYieldExt; @@ -68,6 +71,8 @@ use std::{ sync::Arc, }; +use super::ports::worker; + mod arc_wrapper; /// The on-chain view of the database used by the [`ReadView`] to fetch on-chain data. @@ -100,16 +105,21 @@ impl ReadDatabase { ) -> Self where OnChain: AtomicView + 'static, - OffChain: AtomicView + 'static, + OffChain: AtomicView + worker::OffChainDatabase + 'static, OnChain::LatestView: OnChainDatabase, OffChain::LatestView: OffChainDatabase, { + // TODO[RC]: This fails with clean DB since GraphQL starts before genesis block is executed. + // TODO[RC]: Maybe do not write metadata when executing genesis, but upon creation of off_chain DB? + // let balances_enabled = off_chain.balances_enabled(); + let balances_enabled = false; + Self { batch_size, genesis_height, on_chain: Box::new(ArcWrapper::new(on_chain)), off_chain: Box::new(ArcWrapper::new(off_chain)), - balances_enabled: false, // TODO[RC]: Read this properly + balances_enabled, } } @@ -123,6 +133,7 @@ impl ReadDatabase { genesis_height: self.genesis_height, on_chain: self.on_chain.latest_view()?, off_chain: self.off_chain.latest_view()?, + balances_enabled: self.balances_enabled, }) } @@ -138,6 +149,7 @@ pub struct ReadView { pub(crate) genesis_height: BlockHeight, pub(crate) on_chain: OnChainView, pub(crate) off_chain: OffChainView, + pub(crate) balances_enabled: bool, } impl ReadView { diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index e92901d2121..c6304959ee1 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -1,6 +1,11 @@ -use std::future; +use std::{ + cmp::Ordering, + collections::HashMap, + future, +}; use crate::fuel_core_graphql_api::database::ReadView; +use asset_query::AssetsQuery; use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::IterDirection, @@ -15,11 +20,11 @@ use fuel_core_types::{ }; use futures::{ stream, + FutureExt, Stream, StreamExt, TryStreamExt, }; -use itertools::Either; use tracing::debug; pub mod asset_query; @@ -46,14 +51,85 @@ impl ReadView { direction: IterDirection, base_asset_id: &'a AssetId, ) -> impl Stream> + 'a { - debug!("Querying balances for {:?}", owner); + if self.balances_enabled { + futures::future::Either::Left(self.balances_with_cache( + owner, + base_asset_id, + direction, + )) + } else { + futures::future::Either::Right(self.balances_without_cache( + owner, + base_asset_id, + direction, + )) + } + } + + fn balances_without_cache<'a>( + &'a self, + owner: &'a Address, + base_asset_id: &'a AssetId, + direction: IterDirection, + ) -> impl Stream> + 'a { + debug!(%owner, "Querying balances without balances cache"); + let query = AssetsQuery::new(owner, None, None, self, base_asset_id); + let stream = query.coins(); + stream + .try_fold( + HashMap::new(), + move |mut amounts_per_asset, coin| async move { + let amount: &mut u64 = amounts_per_asset + .entry(*coin.asset_id(base_asset_id)) + .or_default(); + *amount = amount.saturating_add(coin.amount()); + Ok(amounts_per_asset) + }, + ) + .into_stream() + .try_filter_map(move |amounts_per_asset| async move { + let mut balances = amounts_per_asset + .into_iter() + .map(|(asset_id, amount)| AddressBalance { + owner: *owner, + amount, + asset_id, + }) + .collect::>(); + + balances.sort_by(|l, r| { + if l.asset_id < r.asset_id { + Ordering::Less + } else { + Ordering::Greater + } + }); + + if direction == IterDirection::Reverse { + balances.reverse(); + } + + Ok(Some(futures::stream::iter(balances))) + }) + .map_ok(|stream| stream.map(Ok)) + .try_flatten() + .yield_each(self.batch_size) + } + + fn balances_with_cache<'a>( + &'a self, + owner: &'a Address, + base_asset_id: &AssetId, + direction: IterDirection, + ) -> impl Stream> + 'a { + debug!(%owner, "Querying balances using balances cache"); match self.off_chain.balances(owner, base_asset_id) { Ok(balances) => { let iter = if direction == IterDirection::Reverse { - Either::Left(balances.into_iter().rev()) + itertools::Either::Left(balances.into_iter().rev()) } else { - Either::Right(balances.into_iter()) + itertools::Either::Right(balances.into_iter()) }; stream::iter(iter.map(|(asset_id, amount)| AddressBalance { owner: *owner, From a16ef368be72e9db2867a8b09b1109c2f9831273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 12:16:03 +0100 Subject: [PATCH 051/111] Support both 'new' and 'old' way of querying balance --- crates/fuel-core/src/query/balance.rs | 29 +++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index c6304959ee1..2f4204ab2ce 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -5,7 +5,11 @@ use std::{ }; use crate::fuel_core_graphql_api::database::ReadView; -use asset_query::AssetsQuery; +use asset_query::{ + AssetQuery, + AssetSpendTarget, + AssetsQuery, +}; use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::IterDirection, @@ -36,7 +40,28 @@ impl ReadView { asset_id: AssetId, base_asset_id: AssetId, ) -> StorageResult { - let amount = self.off_chain.balance(&owner, &asset_id, &base_asset_id)?; + let amount = if self.balances_enabled { + debug!(%owner, %asset_id, "Querying balance with balances cache"); + self.off_chain.balance(&owner, &asset_id, &base_asset_id)? + } else { + debug!(%owner, %asset_id, "Querying balance without balances cache"); + AssetQuery::new( + &owner, + &AssetSpendTarget::new(asset_id, u64::MAX, u16::MAX), + &base_asset_id, + None, + self, + ) + .coins() + .map(|res| res.map(|coins| coins.amount())) + .try_fold(0u64, |balance, amount| { + async move { + // Increase the balance + Ok(balance.saturating_add(amount)) + } + }) + .await? + }; Ok(AddressBalance { owner, From 80fb8524aab55cd4a676392e294124d5b8cd52c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 13:35:48 +0100 Subject: [PATCH 052/111] Do not touch the onchain DB metadata --- crates/fuel-core/src/database.rs | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 77cf95fb868..b109e20cb79 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -63,6 +63,7 @@ use std::{ fmt::Debug, sync::Arc, }; +use tracing::error; pub use fuel_core_database::Error; pub type Result = core::result::Result; @@ -482,32 +483,13 @@ where ConflictPolicy::Overwrite, changes, ); - - // TODO[RC]: Problems with this code: - // 1. additional DB read (of prev metadata) - // 2. HashSet collect for Metadata V2 - let current_metadata = transaction - .storage::>() - .get(&()) - .map_err(StorageError::from)? - .unwrap(); - let indexation_availability = match current_metadata.as_ref() { - DatabaseMetadata::V1 { version, height } => HashSet::new(), - DatabaseMetadata::V2 { - version, - height, - indexation_availability, - } => [(IndexationKind::Balances)].into_iter().collect(), - }; - transaction .storage_as_mut::>() .insert( &(), - &DatabaseMetadata::V2 { + &DatabaseMetadata::V1 { version: Description::version(), height: new_height, - indexation_availability, }, )?; @@ -527,7 +509,6 @@ where Ok(()) } - #[cfg(feature = "rocksdb")] pub fn convert_to_rocksdb_direction(direction: IterDirection) -> rocksdb::Direction { match direction { From c94a4847899e17a27b0ee82c2110fdc4bd9a0285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 13:38:16 +0100 Subject: [PATCH 053/111] Write balances cache information do off-chain db on genesis --- crates/fuel-core/src/service.rs | 41 ++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 6b42dd3960c..0111107ad51 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -4,7 +4,16 @@ use crate::{ CombinedDatabase, ShutdownListener, }, - database::Database, + database::{ + database_description::{ + off_chain::OffChain, + DatabaseDescription, + DatabaseMetadata, + IndexationKind, + }, + metadata::MetadataTable, + Database, + }, service::{ adapters::{ ExecutorAdapter, @@ -43,6 +52,7 @@ use std::{ net::SocketAddr, sync::Arc, }; +use tracing::info; pub use config::{ Config, @@ -132,6 +142,8 @@ impl FuelService { shutdown_listener, )?; + Self::write_metadata_at_genesis(&database); + // initialize sub services tracing::info!("Initializing sub services"); database.sync_aux_db_heights(shutdown_listener)?; @@ -209,6 +221,33 @@ impl FuelService { Ok(()) } + // When genesis is missing write to the database that balances cache should be used. + fn write_metadata_at_genesis(database: &CombinedDatabase) -> anyhow::Result<()> { + let on_chain_view = database.on_chain().latest_view()?; + if on_chain_view.get_genesis().is_err() { + info!("No genesis, initializing metadata with balances indexation"); + let off_chain_view = database.off_chain().latest_view()?; + let mut database_tx = off_chain_view.read_transaction(); + database_tx + .storage_as_mut::>() + .insert( + &(), + &DatabaseMetadata::V2 { + version: ::version(), + height: Default::default(), + indexation_availability: [(IndexationKind::Balances)] + .into_iter() + .collect(), + }, + )?; + database + .off_chain() + .data + .commit_changes(None, database_tx.into_changes())?; + } + Ok(()) + } + fn make_database_compatible_with_config( combined_database: &mut CombinedDatabase, config: &Config, From ddf0571af77298f78ce5b515ffb3880b1e2d26dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 13:38:43 +0100 Subject: [PATCH 054/111] Use balances cache in GraphQL api --- crates/fuel-core/src/graphql_api/database.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index b7494b888b6..2096b377a18 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -109,10 +109,7 @@ impl ReadDatabase { OnChain::LatestView: OnChainDatabase, OffChain::LatestView: OffChainDatabase, { - // TODO[RC]: This fails with clean DB since GraphQL starts before genesis block is executed. - // TODO[RC]: Maybe do not write metadata when executing genesis, but upon creation of off_chain DB? - // let balances_enabled = off_chain.balances_enabled(); - let balances_enabled = false; + let balances_enabled = off_chain.balances_enabled(); Self { batch_size, From 9078b05b8d30859361f8380abb2e70eb95312a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 13:52:36 +0100 Subject: [PATCH 055/111] Clean-up error handling --- crates/fuel-core/src/database/metadata.rs | 13 ++++++++----- crates/fuel-core/src/graphql_api/api_service.rs | 2 +- crates/fuel-core/src/graphql_api/database.rs | 8 ++++---- crates/fuel-core/src/graphql_api/ports.rs | 4 ++-- crates/fuel-core/src/graphql_api/worker_service.rs | 2 +- .../src/service/adapters/graphql_api/off_chain.rs | 6 ++---- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index e538a863be5..dc047cb74fc 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -20,6 +20,8 @@ use fuel_core_storage::{ StorageMutate, }; +use super::database_description::IndexationKind; + /// The table that stores all metadata about the database. pub struct MetadataTable(core::marker::PhantomData); @@ -78,10 +80,11 @@ where Ok(metadata) } - // TODO[RC]: Needed? - pub fn metadata( - &self, - ) -> StorageResult>>> { - self.storage::>().get(&()) + pub fn indexation_available(&self, kind: IndexationKind) -> StorageResult { + let Some(metadata) = self.storage::>().get(&())? + else { + return Ok(false) + }; + Ok(metadata.indexation_available(kind)) } } diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index dec8883a503..b7e7ba35997 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -242,7 +242,7 @@ where genesis_block_height, on_database, off_database, - ); + )?; let request_timeout = config.config.api_request_timeout; let concurrency_limit = config.config.max_concurrent_queries; let body_limit = config.config.request_body_bytes_limit; diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 2096b377a18..c3c6d22d315 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -102,22 +102,22 @@ impl ReadDatabase { genesis_height: BlockHeight, on_chain: OnChain, off_chain: OffChain, - ) -> Self + ) -> Result where OnChain: AtomicView + 'static, OffChain: AtomicView + worker::OffChainDatabase + 'static, OnChain::LatestView: OnChainDatabase, OffChain::LatestView: OffChainDatabase, { - let balances_enabled = off_chain.balances_enabled(); + let balances_enabled = off_chain.balances_enabled()?; - Self { + Ok(Self { batch_size, genesis_height, on_chain: Box::new(ArcWrapper::new(on_chain)), off_chain: Box::new(ArcWrapper::new(off_chain)), balances_enabled, - } + }) } /// Creates a consistent view of the database. diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 27c509e3d50..f460182279e 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -341,8 +341,8 @@ pub mod worker { /// Creates a write database transaction. fn transaction(&mut self) -> Self::Transaction<'_>; - /// Checks if Balances cache table is available. - fn balances_enabled(&self) -> bool; + /// Checks if Balances cache functionality is available. + fn balances_enabled(&self) -> StorageResult; } pub trait OffChainDatabaseTransaction: diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 400fa819220..a8ef147762a 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -591,7 +591,7 @@ where graphql_metrics().total_txs_count.set(total_tx_count as i64); } - let balances_enabled = self.off_chain_database.balances_enabled(); + let balances_enabled = self.off_chain_database.balances_enabled()?; info!("Balances cache available: {}", balances_enabled); let InitializeTask { diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 677247bee30..ba9f1c444da 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -280,9 +280,7 @@ impl worker::OffChainDatabase for Database { self.into_transaction() } - fn balances_enabled(&self) -> bool { - let metadata = self.metadata().unwrap().unwrap(); // TODO[RC]: Clean-up - error!(?metadata, "METADATA"); - metadata.indexation_available(IndexationKind::Balances) + fn balances_enabled(&self) -> StorageResult { + self.indexation_available(IndexationKind::Balances) } } From 3b6b8c692205bdd41d0a92b62f5675372a6607f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 14:09:09 +0100 Subject: [PATCH 056/111] Satisfy clippy and fmt --- crates/fuel-core/src/coins_query.rs | 1 + crates/fuel-core/src/database.rs | 3 - .../src/database/database_description.rs | 6 +- crates/fuel-core/src/database/metadata.rs | 3 - crates/fuel-core/src/graphql_api/database.rs | 13 +- crates/fuel-core/src/graphql_api/ports.rs | 5 - .../src/graphql_api/worker_service.rs | 5 - .../src/graphql_api/worker_service/tests.rs | 1 + crates/fuel-core/src/lib.rs | 2 +- crates/fuel-core/src/service.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 5 +- crates/fuel-core/src/service/genesis.rs | 3 - .../src/service/genesis/importer/off_chain.rs | 1 - tests/tests/balances.rs | 237 ------------------ 14 files changed, 11 insertions(+), 276 deletions(-) diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index 3d7abda01bc..cc93928b5ed 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -986,6 +986,7 @@ mod tests { let on_chain = self.database.on_chain().clone(); let off_chain = self.database.off_chain().clone(); ServiceDatabase::new(100, 0u32.into(), on_chain, off_chain) + .expect("should create service database") } } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index b109e20cb79..e0acfa9124c 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -24,7 +24,6 @@ use crate::{ KeyValueView, }, }; -use database_description::IndexationKind; use fuel_core_chain_config::TableEntry; use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; use fuel_core_services::SharedMutex; @@ -59,11 +58,9 @@ use fuel_core_types::{ }; use itertools::Itertools; use std::{ - collections::HashSet, fmt::Debug, sync::Arc, }; -use tracing::error; pub use fuel_core_database::Error; pub type Result = core::result::Result; diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 9fe214d1478..4479c245d44 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -4,11 +4,7 @@ use fuel_core_types::{ blockchain::primitives::DaBlockHeight, fuel_types::BlockHeight, }; -use off_chain::OffChain; -use std::collections::{ - HashMap, - HashSet, -}; +use std::collections::HashSet; pub mod gas_price; pub mod off_chain; diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index dc047cb74fc..900a484a16c 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use crate::database::{ database_description::{ DatabaseDescription, @@ -17,7 +15,6 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsRef, StorageInspect, - StorageMutate, }; use super::database_description::IndexationKind; diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index c3c6d22d315..6feaaabc5dc 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -1,11 +1,8 @@ -use crate::{ - database::database_description::DatabaseDescription, - fuel_core_graphql_api::{ - database::arc_wrapper::ArcWrapper, - ports::{ - OffChainDatabase, - OnChainDatabase, - }, +use crate::fuel_core_graphql_api::{ + database::arc_wrapper::ArcWrapper, + ports::{ + OffChainDatabase, + OnChainDatabase, }, }; use fuel_core_services::yield_stream::StreamYieldExt; diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index f460182279e..b87b37603f0 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -280,10 +280,6 @@ pub trait MemoryPool { pub mod worker { use super::super::storage::blocks::FuelBlockIdsToHeights; use crate::{ - database::{ - database_description::off_chain::OffChain, - metadata::MetadataTable, - }, fuel_core_graphql_api::storage::{ coins::OwnedCoins, contracts::ContractsInfo, @@ -310,7 +306,6 @@ pub mod worker { use fuel_core_storage::{ Error as StorageError, Result as StorageResult, - StorageInspect, StorageMutate, }; use fuel_core_types::{ diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index a8ef147762a..4b6bce787a3 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -13,10 +13,6 @@ use super::{ }, }; use crate::{ - database::{ - database_description::off_chain::OffChain, - metadata::MetadataTable, - }, fuel_core_graphql_api::{ ports::{ self, @@ -112,7 +108,6 @@ use std::{ }; use tracing::{ debug, - error, info, }; diff --git a/crates/fuel-core/src/graphql_api/worker_service/tests.rs b/crates/fuel-core/src/graphql_api/worker_service/tests.rs index 8b9ad758975..123501baabb 100644 --- a/crates/fuel-core/src/graphql_api/worker_service/tests.rs +++ b/crates/fuel-core/src/graphql_api/worker_service/tests.rs @@ -83,5 +83,6 @@ fn worker_task_with_block_importer_and_db( chain_id, da_compression_config: DaCompressionConfig::Disabled, continue_on_error: false, + balances_enabled: true, } } diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index 8abd34e1f3d..40d866a137d 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![allow(warnings)] +#![deny(warnings)] use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 0111107ad51..6a654fd406d 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -142,7 +142,7 @@ impl FuelService { shutdown_listener, )?; - Self::write_metadata_at_genesis(&database); + Self::write_metadata_at_genesis(&database)?; // initialize sub services tracing::info!("Initializing sub services"); diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index ba9f1c444da..3329935802f 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -78,10 +78,7 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; -use tracing::{ - debug, - error, -}; +use tracing::debug; impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index 27a74473854..c770d6695ea 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use self::importer::SnapshotImporter; use crate::{ combined_database::{ @@ -20,7 +18,6 @@ use crate::{ }, service::config::Config, }; -use async_graphql::Description; use fuel_core_chain_config::GenesisCommitment; use fuel_core_services::StateWatcher; use fuel_core_storage::{ diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index cedcfbf6e41..cce5d8f5e24 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -1,7 +1,6 @@ use crate::{ database::{ database_description::off_chain::OffChain, - metadata::MetadataTable, GenesisDatabase, }, fuel_core_graphql_api::storage::messages::SpentMessages, diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index d8a6a36da14..a5892b434eb 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -32,7 +32,6 @@ use fuel_core_types::{ TransactionBuilder, }, }; -use hex::FromHex; #[tokio::test] async fn balance() { @@ -230,239 +229,3 @@ async fn first_5_balances() { assert_eq!(balances[i].amount, 300); } } - -// TODO[RC]: This is a temporary test that I used to debug the balance issue. -// I keep it here for now, but it should be removed when the balances caching is finished. -// Or make it more concise and use for thorough balance testing. -#[tokio::test] -async fn foo_2() { - let owner = Address::default(); - let asset_id = AssetId::BASE; - - let different_asset_id = - Vec::from_hex("0606060606060606060606060606060606060606060606060606060606060606") - .unwrap(); - let arr: [u8; 32] = different_asset_id.try_into().unwrap(); - let different_asset_id = AssetId::new(arr); - - // setup config - let mut coin_generator = CoinConfigGenerator::new(); - let state_config = StateConfig { - contracts: vec![], - coins: vec![(owner, 20, asset_id), (owner, 100, different_asset_id)] - .into_iter() - .map(|(owner, amount, asset_id)| CoinConfig { - owner, - amount, - asset_id, - ..coin_generator.generate() - }) - .collect(), - messages: vec![(owner, 10)] - .into_iter() - .enumerate() - .map(|(nonce, (owner, amount))| MessageConfig { - sender: owner, - recipient: owner, - nonce: (nonce as u64).into(), - amount, - data: vec![], - da_height: DaBlockHeight::from(0usize), - }) - .collect(), - ..Default::default() - }; - let config = Config::local_node_with_state_config(state_config); - - // setup server & client - let srv = FuelService::new_node(config).await.unwrap(); - let client = FuelClient::from(srv.bound_address); - - // run test - let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 30); - let balance = client - .balance(&owner, Some(&different_asset_id)) - .await - .unwrap(); - assert_eq!(balance, 100); - - println!( - "INIT PHASE COMPLETE, will be now sending 1 '0000...' coin to user '0101...'" - ); - - // spend DIFFERENT COIN and check again - { - let coins_per_asset = client - .coins_to_spend(&owner, vec![(different_asset_id, 1, None)], None) - .await - .unwrap(); - println!("coins to spend = {:#?}", &coins_per_asset); - let mut tx = TransactionBuilder::script(vec![], vec![]) - .script_gas_limit(1_000_000) - .to_owned(); - for coins in coins_per_asset { - for coin in coins { - match coin { - CoinType::Coin(coin) => tx.add_input(Input::coin_signed( - coin.utxo_id, - coin.owner, - coin.amount, - coin.asset_id, - Default::default(), - 0, - )), - CoinType::MessageCoin(message) => { - tx.add_input(Input::message_coin_signed( - message.sender, - message.recipient, - message.amount, - message.nonce, - 0, - )) - } - CoinType::Unknown => panic!("Unknown coin"), - }; - } - } - - let tx = tx - .add_output(Output::Coin { - to: Address::new([1u8; 32]), - amount: 2, - asset_id: different_asset_id, - }) - .add_output(Output::Change { - to: owner, - amount: 0, - asset_id: different_asset_id, - }) - .add_witness(Default::default()) - .finalize_as_transaction(); - - client.submit_and_await_commit(&tx).await.unwrap(); - - let balance = client - .balance(&owner, Some(&different_asset_id)) - .await - .unwrap(); - assert_eq!(balance, 98); - } - - // spend COIN and check again - { - let coins_per_asset = client - .coins_to_spend(&owner, vec![(asset_id, 1, None)], None) - .await - .unwrap(); - println!("coins to spend = {:#?}", &coins_per_asset); - let mut tx = TransactionBuilder::script(vec![], vec![]) - .script_gas_limit(1_000_000) - .to_owned(); - for coins in coins_per_asset { - for coin in coins { - match coin { - CoinType::Coin(coin) => tx.add_input(Input::coin_signed( - coin.utxo_id, - coin.owner, - coin.amount, - coin.asset_id, - Default::default(), - 0, - )), - CoinType::MessageCoin(message) => { - tx.add_input(Input::message_coin_signed( - message.sender, - message.recipient, - message.amount, - message.nonce, - 0, - )) - } - CoinType::Unknown => panic!("Unknown coin"), - }; - } - } - - let tx = tx - .add_output(Output::Coin { - to: Address::new([1u8; 32]), - amount: 1, - asset_id, - }) - .add_output(Output::Change { - to: owner, - amount: 0, - asset_id, - }) - .add_witness(Default::default()) - .finalize_as_transaction(); - - client.submit_and_await_commit(&tx).await.unwrap(); - - let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); - assert_eq!(balance, 29); - } - - println!( - "INIT PHASE COMPLETE, will be now sending 2 '0606...' coin to user '0101...'" - ); - - // spend DIFFERENT COIN and check again - { - let coins_per_asset = client - .coins_to_spend(&owner, vec![(different_asset_id, 1, None)], None) - .await - .unwrap(); - println!("coins to spend = {:#?}", &coins_per_asset); - let mut tx = TransactionBuilder::script(vec![], vec![]) - .script_gas_limit(1_000_000) - .to_owned(); - for coins in coins_per_asset { - for coin in coins { - match coin { - CoinType::Coin(coin) => tx.add_input(Input::coin_signed( - coin.utxo_id, - coin.owner, - coin.amount, - coin.asset_id, - Default::default(), - 0, - )), - CoinType::MessageCoin(message) => { - tx.add_input(Input::message_coin_signed( - message.sender, - message.recipient, - message.amount, - message.nonce, - 0, - )) - } - CoinType::Unknown => panic!("Unknown coin"), - }; - } - } - - let tx = tx - .add_output(Output::Coin { - to: Address::new([1u8; 32]), - amount: 3, - asset_id: different_asset_id, - }) - .add_output(Output::Change { - to: owner, - amount: 0, - asset_id: different_asset_id, - }) - .add_witness(Default::default()) - .finalize_as_transaction(); - - client.submit_and_await_commit(&tx).await.unwrap(); - - let balance = client - .balance(&owner, Some(&different_asset_id)) - .await - .unwrap(); - assert_eq!(balance, 95); - } -} From 51629368740529f608a2f65e552d213a268fe339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 17:24:19 +0100 Subject: [PATCH 057/111] generic update function --- .../src/database/database_description.rs | 2 +- crates/fuel-core/src/graphql_api.rs | 2 +- .../src/graphql_api/worker_service.rs | 164 ++++++++++-------- crates/fuel-core/src/lib.rs | 2 +- 4 files changed, 94 insertions(+), 76 deletions(-) diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 4479c245d44..9d6c38ead3d 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -73,7 +73,7 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { )] pub enum IndexationKind { Balances, - CoinsToSpend, + _CoinsToSpend, } /// The metadata of the database contains information about the version and its height. diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 24e70c1dcbf..0dc3fe6b8db 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -79,7 +79,7 @@ impl Default for Costs { } pub const DEFAULT_QUERY_COSTS: Costs = Costs { - balance_query: 40001, /* TODO[RC]: We might consider making this query cheaper because it's just a single DB read now. */ + balance_query: 40001, /* Cost will depend on whether balances index is available or not, but let's keep the default high to be on the safe side */ coins_to_spend: 40001, get_peers: 40001, estimate_predicates: 40001, diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 4b6bce787a3..3567c38728a 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -49,8 +49,12 @@ use fuel_core_services::{ }; use fuel_core_storage::{ Error as StorageError, + Mappable, Result as StorageResult, StorageAsMut, + StorageInspect, + StorageMut, + StorageMutate, }; use fuel_core_types::{ blockchain::{ @@ -60,7 +64,11 @@ use fuel_core_types::{ }, consensus::Consensus, }, - entities::relayer::transaction::RelayedTransactionStatus, + entities::{ + coins::coin::Coin, + relayer::transaction::RelayedTransactionStatus, + Message, + }, fuel_tx::{ field::{ Inputs, @@ -199,58 +207,71 @@ where } } -fn update_coin_balance( - owner: &Address, - asset_id: &AssetId, - amount: Amount, - tx: &mut T, - updater: F, -) -> StorageResult<()> -where - T: OffChainDatabaseTransaction, - F: Fn(Cow, Amount) -> Amount, -{ - let key = BalancesKey::new(owner, asset_id); - let current_balance = tx.storage::().get(&key)?.unwrap_or_default(); - let prev_balance = current_balance.clone(); - let new_balance = updater(current_balance, amount); - debug!( - %owner, - %asset_id, - amount, - %prev_balance, - new_balance, - "changing coin balance" - ); - tx.storage_as_mut::() - .insert(&key, &new_balance) +trait HasIndexation<'a> { + type Storage: Mappable; + + fn key(&self) -> ::Key; + fn amount(&self) -> Amount; + fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> + where + <>::Storage as Mappable>::Key: Sized + std::fmt::Debug, + <>::Storage as Mappable>::Value: + Sized + std::fmt::Display, + <>::Storage as Mappable>::OwnedValue: + Default + std::fmt::Display, + T: OffChainDatabaseTransaction + + StorageInspect< + >::Storage, + Error = fuel_core_storage::Error, + > + StorageMutate<>::Storage>, + F: Fn( + Cow<<>::Storage as Mappable>::OwnedValue>, + Amount, + ) -> <>::Storage as Mappable>::Value, + fuel_core_storage::Error: + From<>::Storage>>::Error>, + { + let key = self.key(); + let amount = self.amount(); + let storage = tx.storage::(); + let current_balance = storage.get(&key)?.unwrap_or_default(); + let prev_balance = current_balance.clone(); + let new_balance = updater(current_balance, amount); + + debug!( + ?key, + %amount, + %prev_balance, + %new_balance, + "changing balance"); + + let storage = tx.storage::(); + storage.insert(&key, &new_balance) + } } -fn update_message_balance( - owner: &Address, - amount: Amount, - tx: &mut T, - updater: F, -) -> StorageResult<()> -where - T: OffChainDatabaseTransaction, - F: Fn(Cow, Amount) -> Amount, -{ - let key = owner; - let current_balance = tx - .storage::() - .get(key)? - .unwrap_or_default(); - let prev_balance = current_balance.clone(); - let new_balance = updater(current_balance, amount); - debug!( - %owner, - %amount, - %prev_balance, - new_balance, - "changing message balance"); - tx.storage_as_mut::() - .insert(key, &new_balance) +impl<'a> HasIndexation<'a> for Coin { + type Storage = CoinBalances; + + fn key(&self) -> ::Key { + BalancesKey::new(&self.owner, &self.asset_id) + } + + fn amount(&self) -> Amount { + self.amount + } +} + +impl<'a> HasIndexation<'a> for Message { + type Storage = MessageBalances; + + fn key(&self) -> ::Key { + self.recipient().clone() + } + + fn amount(&self) -> Amount { + Self::amount(&self) + } } /// Process the executor events and update the indexes for the messages and coins. @@ -275,12 +296,10 @@ where // TODO[RC]: Refactor to have this if called only once if balances_enabled { - update_message_balance( - message.recipient(), - message.amount(), + message.update_balances( block_st_transaction, - |balance, amount| balance.saturating_add(amount), - )?; + |balance: Cow, amount| balance.saturating_add(amount), + ); } } Event::MessageConsumed(message) => { @@ -296,12 +315,10 @@ where // TODO[RC]: Check other places where we update "OwnedCoins" or "OwnedMessageIds" if balances_enabled { - update_message_balance( - message.recipient(), - message.amount(), + message.update_balances( block_st_transaction, - |balance, amount| balance.saturating_sub(amount), - )?; + |balance: Cow, amount| balance.saturating_sub(amount), + ); } } Event::CoinCreated(coin) => { @@ -311,13 +328,17 @@ where .insert(&coin_by_owner, &())?; if balances_enabled { - update_coin_balance( - &coin.owner, - &coin.asset_id, - coin.amount, + coin.update_balances( block_st_transaction, - |balance, amount| balance.saturating_add(amount), - )?; + |balance: Cow, amount| balance.saturating_add(amount), + ); + // update_coin_balance( + // &coin.owner, + // &coin.asset_id, + // coin.amount, + // block_st_transaction, + // |balance, amount| balance.saturating_add(amount), + // )?; } } Event::CoinConsumed(coin) => { @@ -327,13 +348,10 @@ where .remove(&key)?; if balances_enabled { - update_coin_balance( - &coin.owner, - &coin.asset_id, - coin.amount, + coin.update_balances( block_st_transaction, - |balance, amount| balance.saturating_sub(amount), - )?; + |balance: Cow, amount| balance.saturating_sub(amount), + ); } } Event::ForcedTransactionFailed { diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index 40d866a137d..8abd34e1f3d 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![deny(warnings)] +#![allow(warnings)] use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; From f736ecbd8f94c8f06aec00a1a361523d85020b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 28 Oct 2024 17:32:25 +0100 Subject: [PATCH 058/111] Extract processing of balance updates --- .../src/graphql_api/worker_service.rs | 73 +++++++++---------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 3567c38728a..dc5ac5cc74c 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -274,6 +274,41 @@ impl<'a> HasIndexation<'a> for Message { } } +fn process_balances_update( + event: &Event, + block_st_transaction: &mut T, + balances_enabled: bool, +) where + T: OffChainDatabaseTransaction, +{ + if !balances_enabled { + return; + } + match event.deref() { + Event::MessageImported(message) => { + message.update_balances(block_st_transaction, |balance: Cow, amount| { + balance.saturating_add(amount) + }); + } + Event::MessageConsumed(message) => { + message.update_balances(block_st_transaction, |balance: Cow, amount| { + balance.saturating_sub(amount) + }); + } + Event::CoinCreated(coin) => { + coin.update_balances(block_st_transaction, |balance: Cow, amount| { + balance.saturating_add(amount) + }); + } + Event::CoinConsumed(coin) => { + coin.update_balances(block_st_transaction, |balance: Cow, amount| { + balance.saturating_sub(amount) + }); + } + _ => {} + } +} + /// Process the executor events and update the indexes for the messages and coins. pub fn process_executor_events<'a, Iter, T>( events: Iter, @@ -285,6 +320,7 @@ where T: OffChainDatabaseTransaction, { for event in events { + process_balances_update(event.deref(), block_st_transaction, balances_enabled); match event.deref() { Event::MessageImported(message) => { block_st_transaction @@ -293,14 +329,6 @@ where &OwnedMessageKey::new(message.recipient(), message.nonce()), &(), )?; - - // TODO[RC]: Refactor to have this if called only once - if balances_enabled { - message.update_balances( - block_st_transaction, - |balance: Cow, amount| balance.saturating_add(amount), - ); - } } Event::MessageConsumed(message) => { block_st_transaction @@ -312,47 +340,18 @@ where block_st_transaction .storage::() .insert(message.nonce(), &())?; - - // TODO[RC]: Check other places where we update "OwnedCoins" or "OwnedMessageIds" - if balances_enabled { - message.update_balances( - block_st_transaction, - |balance: Cow, amount| balance.saturating_sub(amount), - ); - } } Event::CoinCreated(coin) => { let coin_by_owner = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .insert(&coin_by_owner, &())?; - - if balances_enabled { - coin.update_balances( - block_st_transaction, - |balance: Cow, amount| balance.saturating_add(amount), - ); - // update_coin_balance( - // &coin.owner, - // &coin.asset_id, - // coin.amount, - // block_st_transaction, - // |balance, amount| balance.saturating_add(amount), - // )?; - } } Event::CoinConsumed(coin) => { let key = owner_coin_id_key(&coin.owner, &coin.utxo_id); block_st_transaction .storage_as_mut::() .remove(&key)?; - - if balances_enabled { - coin.update_balances( - block_st_transaction, - |balance: Cow, amount| balance.saturating_sub(amount), - ); - } } Event::ForcedTransactionFailed { id, From 6927f03dd7ced0250cd3cf392c6f5c87e5edce03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 29 Oct 2024 10:12:16 +0100 Subject: [PATCH 059/111] Use instead of for balances key --- crates/fuel-core/src/graphql_api/storage/balances.rs | 6 ++++++ crates/fuel-core/src/graphql_api/worker_service.rs | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 20dd21de3b8..8415964521a 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -31,6 +31,12 @@ impl Distribution for Standard { } } +impl core::fmt::Display for BalancesKey { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "address={} asset_id={}", self.address(), self.asset_id()) + } +} + /// This table stores the balances of coins per owner and asset id. pub struct CoinBalances; diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index dc5ac5cc74c..a4a9a82d885 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -214,7 +214,8 @@ trait HasIndexation<'a> { fn amount(&self) -> Amount; fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> where - <>::Storage as Mappable>::Key: Sized + std::fmt::Debug, + <>::Storage as Mappable>::Key: + Sized + std::fmt::Display, <>::Storage as Mappable>::Value: Sized + std::fmt::Display, <>::Storage as Mappable>::OwnedValue: @@ -239,7 +240,7 @@ trait HasIndexation<'a> { let new_balance = updater(current_balance, amount); debug!( - ?key, + %key, %amount, %prev_balance, %new_balance, From 66e26d958cfe838adff1ef8af4748d5b953e3b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 29 Oct 2024 15:08:03 +0100 Subject: [PATCH 060/111] Revert the supporsed-to-be temporary change that disabled warnings --- .../src/graphql_api/worker_service.rs | 46 ++++++++----------- crates/fuel-core/src/lib.rs | 2 +- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index a4a9a82d885..d056fcfd2b5 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -53,7 +53,6 @@ use fuel_core_storage::{ Result as StorageResult, StorageAsMut, StorageInspect, - StorageMut, StorageMutate, }; use fuel_core_types::{ @@ -80,8 +79,6 @@ use fuel_core_types::{ CoinPredicate, CoinSigned, }, - Address, - AssetId, Contract, Input, Output, @@ -267,11 +264,11 @@ impl<'a> HasIndexation<'a> for Message { type Storage = MessageBalances; fn key(&self) -> ::Key { - self.recipient().clone() + *self.recipient() } fn amount(&self) -> Amount { - Self::amount(&self) + Self::amount(self) } } @@ -279,34 +276,31 @@ fn process_balances_update( event: &Event, block_st_transaction: &mut T, balances_enabled: bool, -) where +) -> StorageResult<()> +where T: OffChainDatabaseTransaction, { if !balances_enabled { - return; + return Ok(()); } - match event.deref() { - Event::MessageImported(message) => { - message.update_balances(block_st_transaction, |balance: Cow, amount| { + match event { + Event::MessageImported(message) => message + .update_balances(block_st_transaction, |balance: Cow, amount| { balance.saturating_add(amount) - }); - } - Event::MessageConsumed(message) => { - message.update_balances(block_st_transaction, |balance: Cow, amount| { + }), + Event::MessageConsumed(message) => message + .update_balances(block_st_transaction, |balance: Cow, amount| { balance.saturating_sub(amount) - }); - } - Event::CoinCreated(coin) => { - coin.update_balances(block_st_transaction, |balance: Cow, amount| { + }), + Event::CoinCreated(coin) => coin + .update_balances(block_st_transaction, |balance: Cow, amount| { balance.saturating_add(amount) - }); - } - Event::CoinConsumed(coin) => { - coin.update_balances(block_st_transaction, |balance: Cow, amount| { + }), + Event::CoinConsumed(coin) => coin + .update_balances(block_st_transaction, |balance: Cow, amount| { balance.saturating_sub(amount) - }); - } - _ => {} + }), + Event::ForcedTransactionFailed { .. } => Ok(()), } } @@ -321,7 +315,7 @@ where T: OffChainDatabaseTransaction, { for event in events { - process_balances_update(event.deref(), block_st_transaction, balances_enabled); + process_balances_update(event.deref(), block_st_transaction, balances_enabled)?; match event.deref() { Event::MessageImported(message) => { block_st_transaction diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index 8abd34e1f3d..40d866a137d 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![allow(warnings)] +#![deny(warnings)] use crate::service::genesis::NotifyCancel; use tokio_util::sync::CancellationToken; From a87698a8edcc11b3e02b3e6dd74e3b7ca16be1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 29 Oct 2024 15:31:33 +0100 Subject: [PATCH 061/111] Remove unused lifetime from `HasIndexation` trait --- .../src/graphql_api/worker_service.rs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index d056fcfd2b5..4b480df9057 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -204,30 +204,28 @@ where } } -trait HasIndexation<'a> { +trait HasIndexation { type Storage: Mappable; fn key(&self) -> ::Key; fn amount(&self) -> Amount; fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> where - <>::Storage as Mappable>::Key: - Sized + std::fmt::Display, - <>::Storage as Mappable>::Value: - Sized + std::fmt::Display, - <>::Storage as Mappable>::OwnedValue: + <::Storage as Mappable>::Key: Sized + std::fmt::Display, + <::Storage as Mappable>::Value: Sized + std::fmt::Display, + <::Storage as Mappable>::OwnedValue: Default + std::fmt::Display, T: OffChainDatabaseTransaction + StorageInspect< - >::Storage, + ::Storage, Error = fuel_core_storage::Error, - > + StorageMutate<>::Storage>, + > + StorageMutate<::Storage>, F: Fn( - Cow<<>::Storage as Mappable>::OwnedValue>, + Cow<<::Storage as Mappable>::OwnedValue>, Amount, - ) -> <>::Storage as Mappable>::Value, + ) -> <::Storage as Mappable>::Value, fuel_core_storage::Error: - From<>::Storage>>::Error>, + From<::Storage>>::Error>, { let key = self.key(); let amount = self.amount(); @@ -248,7 +246,7 @@ trait HasIndexation<'a> { } } -impl<'a> HasIndexation<'a> for Coin { +impl HasIndexation for Coin { type Storage = CoinBalances; fn key(&self) -> ::Key { @@ -260,7 +258,7 @@ impl<'a> HasIndexation<'a> for Coin { } } -impl<'a> HasIndexation<'a> for Message { +impl HasIndexation for Message { type Storage = MessageBalances; fn key(&self) -> ::Key { From ddadd8030beb5c67bd4c861206fd524148c7b68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 29 Oct 2024 15:54:16 +0100 Subject: [PATCH 062/111] Simplify trait bounds for `HasIndexation` trait --- .../src/graphql_api/worker_service.rs | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 4b480df9057..61c3b73dde2 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -211,21 +211,17 @@ trait HasIndexation { fn amount(&self) -> Amount; fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> where - <::Storage as Mappable>::Key: Sized + std::fmt::Display, - <::Storage as Mappable>::Value: Sized + std::fmt::Display, - <::Storage as Mappable>::OwnedValue: - Default + std::fmt::Display, + ::Key: Sized + std::fmt::Display, + ::Value: Sized + std::fmt::Display, + ::OwnedValue: Default + std::fmt::Display, T: OffChainDatabaseTransaction - + StorageInspect< - ::Storage, - Error = fuel_core_storage::Error, - > + StorageMutate<::Storage>, + + StorageInspect + + StorageMutate, F: Fn( - Cow<<::Storage as Mappable>::OwnedValue>, + Cow<::OwnedValue>, Amount, - ) -> <::Storage as Mappable>::Value, - fuel_core_storage::Error: - From<::Storage>>::Error>, + ) -> ::Value, + fuel_core_storage::Error: From<>::Error>, { let key = self.key(); let amount = self.amount(); From ccadf4169694861cefff82d3aae15d3ff32a3456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 29 Oct 2024 16:01:53 +0100 Subject: [PATCH 063/111] Further simplify trait bounds for `HasIndexation` trait --- crates/fuel-core/src/graphql_api/worker_service.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 61c3b73dde2..f34dda92ed1 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -214,9 +214,7 @@ trait HasIndexation { ::Key: Sized + std::fmt::Display, ::Value: Sized + std::fmt::Display, ::OwnedValue: Default + std::fmt::Display, - T: OffChainDatabaseTransaction - + StorageInspect - + StorageMutate, + T: OffChainDatabaseTransaction + StorageMutate, F: Fn( Cow<::OwnedValue>, Amount, @@ -238,7 +236,7 @@ trait HasIndexation { "changing balance"); let storage = tx.storage::(); - storage.insert(&key, &new_balance) + Ok(storage.insert(&key, &new_balance)?) } } From 91e8abc6c8cd67a22b0119bf9f54c25fd4e6b168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 30 Oct 2024 16:04:27 +0100 Subject: [PATCH 064/111] Split and rename the `HasIndexation` trait --- .../src/graphql_api/worker_service.rs | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index f34dda92ed1..e37ad8db467 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -204,11 +204,38 @@ where } } -trait HasIndexation { +trait DatabaseItemWithAmount { type Storage: Mappable; fn key(&self) -> ::Key; fn amount(&self) -> Amount; +} + +impl DatabaseItemWithAmount for Coin { + type Storage = CoinBalances; + + fn key(&self) -> ::Key { + BalancesKey::new(&self.owner, &self.asset_id) + } + + fn amount(&self) -> Amount { + self.amount + } +} + +impl DatabaseItemWithAmount for Message { + type Storage = MessageBalances; + + fn key(&self) -> ::Key { + *self.recipient() + } + + fn amount(&self) -> Amount { + Self::amount(self) + } +} + +trait BalanceIndexationUpdater: DatabaseItemWithAmount { fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> where ::Key: Sized + std::fmt::Display, @@ -240,29 +267,8 @@ trait HasIndexation { } } -impl HasIndexation for Coin { - type Storage = CoinBalances; - - fn key(&self) -> ::Key { - BalancesKey::new(&self.owner, &self.asset_id) - } - - fn amount(&self) -> Amount { - self.amount - } -} - -impl HasIndexation for Message { - type Storage = MessageBalances; - - fn key(&self) -> ::Key { - *self.recipient() - } - - fn amount(&self) -> Amount { - Self::amount(self) - } -} +impl BalanceIndexationUpdater for Coin {} +impl BalanceIndexationUpdater for Message {} fn process_balances_update( event: &Event, From ab7808f540475ec25a57953159b190c3477c0fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 10:11:35 +0100 Subject: [PATCH 065/111] WIP - bail when unable to update balance due to overflow --- .../src/graphql_api/worker_service.rs | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index e37ad8db467..3c88104b566 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -113,6 +113,7 @@ use std::{ }; use tracing::{ debug, + error, info, }; @@ -245,7 +246,7 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { F: Fn( Cow<::OwnedValue>, Amount, - ) -> ::Value, + ) -> Option<::Value>, fuel_core_storage::Error: From<>::Error>, { let key = self.key(); @@ -253,17 +254,27 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { let storage = tx.storage::(); let current_balance = storage.get(&key)?.unwrap_or_default(); let prev_balance = current_balance.clone(); - let new_balance = updater(current_balance, amount); - - debug!( - %key, - %amount, - %prev_balance, - %new_balance, - "changing balance"); - - let storage = tx.storage::(); - Ok(storage.insert(&key, &new_balance)?) + match updater(current_balance, amount) { + Some(new_balance) => { + debug!( + %key, + %amount, + %prev_balance, + %new_balance, + "changing balance"); + + let storage = tx.storage::(); + Ok(storage.insert(&key, &new_balance)?) + } + None => { + error!( + %key, + %amount, + %prev_balance, + "unable to change balance due to overflow"); + Err(anyhow::anyhow!("unable to change balance due to overflow").into()) + } + } } } @@ -284,19 +295,19 @@ where match event { Event::MessageImported(message) => message .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.saturating_add(amount) + balance.checked_add(amount) }), Event::MessageConsumed(message) => message .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.saturating_sub(amount) + balance.checked_sub(amount) }), Event::CoinCreated(coin) => coin .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.saturating_add(amount) + balance.checked_add(amount) }), Event::CoinConsumed(coin) => coin .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.saturating_sub(amount) + balance.checked_sub(amount) }), Event::ForcedTransactionFailed { .. } => Ok(()), } From 90fa259606b846db2fe0432a3679a74b017b681c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 12:56:09 +0100 Subject: [PATCH 066/111] Prefer `core::fmt::Display` over `std::fmt::Display` --- crates/fuel-core/src/graphql_api/worker_service.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 3c88104b566..5829d12ec8f 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -239,9 +239,9 @@ impl DatabaseItemWithAmount for Message { trait BalanceIndexationUpdater: DatabaseItemWithAmount { fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> where - ::Key: Sized + std::fmt::Display, - ::Value: Sized + std::fmt::Display, - ::OwnedValue: Default + std::fmt::Display, + ::Key: Sized + core::fmt::Display, + ::Value: Sized + core::fmt::Display, + ::OwnedValue: Default + core::fmt::Display, T: OffChainDatabaseTransaction + StorageMutate, F: Fn( Cow<::OwnedValue>, From 0b39319e78506c9502e5325062f77f84f41f5a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 17:04:54 +0100 Subject: [PATCH 067/111] Use named const instead of plain `true` in genesis importer for clarity --- .../src/service/genesis/importer/off_chain.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index cce5d8f5e24..3b7607c738c 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -107,10 +107,13 @@ impl ImportTable for Handler { group: Vec>, tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { + // We always want to enable balances indexation if we're starting at genesis. + const BALANCES_INDEXATION_ENABLED: bool = true; + let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - worker_service::process_executor_events(events, tx, true)?; + worker_service::process_executor_events(events, tx, BALANCES_INDEXATION_ENABLED)?; Ok(()) } } @@ -125,10 +128,13 @@ impl ImportTable for Handler { group: Vec>, tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { + // We always want to enable balances indexation if we're starting at genesis. + const BALANCES_INDEXATION_ENABLED: bool = true; + let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - worker_service::process_executor_events(events, tx, true)?; + worker_service::process_executor_events(events, tx, BALANCES_INDEXATION_ENABLED)?; Ok(()) } } From 4d6e17a84543a6bf035848cc844077a15710ad4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 17:17:24 +0100 Subject: [PATCH 068/111] Remove reduntant write to `DatabaseMetadata::V2` --- crates/fuel-core/src/service/genesis.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index c770d6695ea..55d36e2821b 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -8,12 +8,8 @@ use crate::{ database_description::{ off_chain::OffChain, on_chain::OnChain, - DatabaseDescription, - DatabaseMetadata, - IndexationKind, }, genesis_progress::GenesisMetadata, - metadata::MetadataTable, Database, }, service::config::Config, @@ -134,19 +130,6 @@ pub async fn execute_genesis_block( .storage_as_mut::>() .remove(&key)?; } - - database_transaction_off_chain - .storage_as_mut::>() - .insert( - &(), - &DatabaseMetadata::V2 { - version: ::version(), - height: Default::default(), - indexation_availability: [(IndexationKind::Balances)] - .into_iter() - .collect(), - }, - )?; database_transaction_off_chain.commit()?; let mut database_transaction_on_chain = db.on_chain().read_transaction(); From 8f3e81700af579b8b570b1c88b23d6d848e89262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 17:40:09 +0100 Subject: [PATCH 069/111] Ensure all indexations are enabled at genesis --- .../src/database/database_description.rs | 18 ++++++++++++++++-- crates/fuel-core/src/service.rs | 10 ++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 9d6c38ead3d..9fb1fc73d5b 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -5,6 +5,7 @@ use fuel_core_types::{ fuel_types::BlockHeight, }; use std::collections::HashSet; +use strum::IntoEnumIterator; pub mod gas_price; pub mod off_chain; @@ -69,11 +70,24 @@ pub trait DatabaseDescription: 'static + Copy + Debug + Send + Sync { } #[derive( - Copy, Clone, Debug, serde::Serialize, serde::Deserialize, Eq, PartialEq, Hash, + Copy, + Clone, + Debug, + serde::Serialize, + serde::Deserialize, + Eq, + PartialEq, + Hash, + strum::EnumIter, )] pub enum IndexationKind { Balances, - _CoinsToSpend, +} + +impl IndexationKind { + pub fn all() -> impl Iterator { + Self::iter() + } } /// The metadata of the database contains information about the version and its height. diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 6a654fd406d..9474d08601f 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -225,7 +225,11 @@ impl FuelService { fn write_metadata_at_genesis(database: &CombinedDatabase) -> anyhow::Result<()> { let on_chain_view = database.on_chain().latest_view()?; if on_chain_view.get_genesis().is_err() { - info!("No genesis, initializing metadata with balances indexation"); + let all_indexations = IndexationKind::all().collect(); + info!( + "No genesis, initializing metadata with all supported indexations: {:?}", + all_indexations + ); let off_chain_view = database.off_chain().latest_view()?; let mut database_tx = off_chain_view.read_transaction(); database_tx @@ -235,9 +239,7 @@ impl FuelService { &DatabaseMetadata::V2 { version: ::version(), height: Default::default(), - indexation_availability: [(IndexationKind::Balances)] - .into_iter() - .collect(), + indexation_availability: all_indexations, }, )?; database From 895d9da0928d3c799ae292d50be1853059cfc359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 31 Oct 2024 19:09:45 +0100 Subject: [PATCH 070/111] Fix `produce_block__raises_gas_price` after balance overflow check was added --- tests/tests/gas_price.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 18507785eec..d92214969a9 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -8,6 +8,7 @@ use fuel_core::{ chain_config::{ ChainConfig, StateConfig, + TESTNET_WALLET_SECRETS, }, database::Database, service::{ @@ -50,6 +51,7 @@ use rand::Rng; use std::{ iter::repeat, ops::Deref, + str::FromStr, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -71,13 +73,17 @@ fn arb_large_tx( let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); let mut builder = TransactionBuilder::script(script_bytes, vec![]); let asset_id = *builder.get_params().base_asset_id(); + let wallet = SecretKey::from_str( + TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())], + ) + .expect("should parse secret key hex bytes"); builder .max_fee_limit(max_fee_limit) .script_gas_limit(22430) .add_unsigned_coin_input( - SecretKey::random(rng), + wallet, rng.gen(), - u32::MAX as u64, + max_fee_limit, asset_id, Default::default(), ) @@ -149,10 +155,12 @@ async fn produce_block__raises_gas_price() { let client = FuelClient::from(srv.bound_address); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + const MAX_FEE: u64 = 189028; + // when let arb_tx_count = 10; for i in 0..arb_tx_count { - let tx = arb_large_tx(189028 + i as Word, &mut rng); + let tx = arb_large_tx(MAX_FEE + i as Word, &mut rng); let _status = client.submit(&tx).await.unwrap(); } // starting gas price From de11071b8dbafb8b6d40048eaf847f5adc28078f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 1 Nov 2024 11:52:07 +0100 Subject: [PATCH 071/111] Fix more integration tests to work with the balance overflow checks --- crates/chain-config/src/config/state.rs | 17 +++++++++++++++++ tests/test-helpers/src/lib.rs | 23 ++++++++++++++--------- tests/tests/blocks.rs | 2 +- tests/tests/gas_price.rs | 17 ++++++----------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/crates/chain-config/src/config/state.rs b/crates/chain-config/src/config/state.rs index c8fee0468fd..f64954dbab3 100644 --- a/crates/chain-config/src/config/state.rs +++ b/crates/chain-config/src/config/state.rs @@ -37,6 +37,12 @@ use fuel_core_types::{ fuel_vm::BlobData, }; use itertools::Itertools; +#[cfg(feature = "test-helpers")] +use rand::{ + CryptoRng, + Rng, + RngCore, +}; use serde::{ Deserialize, Serialize, @@ -87,6 +93,17 @@ pub const TESTNET_WALLET_SECRETS: [&str; 5] = [ "0x7f8a325504e7315eda997db7861c9447f5c3eff26333b20180475d94443a10c6", ]; +#[cfg(feature = "test-helpers")] +pub fn random_testnet_wallet( + rng: &mut (impl CryptoRng + RngCore), +) -> (SecretKey, &'static str) { + let wallet_str = + TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())]; + let wallet = + SecretKey::from_str(wallet_str).expect("should parse secret key hex bytes"); + (wallet, wallet_str) +} + #[derive(Default, Copy, Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct LastBlockConfig { /// The block height of the last block. diff --git a/tests/test-helpers/src/lib.rs b/tests/test-helpers/src/lib.rs index 89e3f0c0616..32f07871fde 100644 --- a/tests/test-helpers/src/lib.rs +++ b/tests/test-helpers/src/lib.rs @@ -1,3 +1,6 @@ +use std::str::FromStr; + +use fuel_core::chain_config::random_testnet_wallet; use fuel_core_client::client::{ types::TransactionStatus, FuelClient, @@ -9,9 +12,11 @@ use fuel_core_types::{ }, fuel_crypto::SecretKey, fuel_tx::{ + Address, Output, Transaction, TransactionBuilder, + Word, }, }; use rand::{ @@ -33,27 +38,27 @@ pub async fn send_graph_ql_query(url: &str, query: &str) -> String { response.text().await.unwrap() } -pub fn make_tx( - rng: &mut (impl CryptoRng + RngCore), - i: u64, - max_gas_limit: u64, -) -> Transaction { +pub fn make_tx(rng: &mut (impl CryptoRng + RngCore), max_gas_limit: u64) -> Transaction { + const SMALL_AMOUNT: Word = 2; + + let (wallet, wallet_str) = random_testnet_wallet(rng); + TransactionBuilder::script( op::ret(RegId::ONE).to_bytes().into_iter().collect(), vec![], ) .script_gas_limit(max_gas_limit / 2) .add_unsigned_coin_input( - SecretKey::random(rng), + wallet, rng.gen(), - 1000 + i, + SMALL_AMOUNT, Default::default(), Default::default(), ) .add_output(Output::Change { - amount: 0, + amount: SMALL_AMOUNT, asset_id: Default::default(), - to: rng.gen(), + to: Address::from_str(wallet_str).expect("should parse bytes as address"), }) .finalize_as_transaction() } diff --git a/tests/tests/blocks.rs b/tests/tests/blocks.rs index 7a5ba4688d5..2fb0e99bb6a 100644 --- a/tests/tests/blocks.rs +++ b/tests/tests/blocks.rs @@ -481,7 +481,7 @@ mod full_block { let tx_count: u64 = 66_000; let txs = (1..=tx_count) - .map(|i| test_helpers::make_tx(&mut rng, i, max_gas_limit)) + .map(|_| test_helpers::make_tx(&mut rng, max_gas_limit)) .collect_vec(); // When diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index d92214969a9..ed0ce66b810 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -6,9 +6,9 @@ use crate::helpers::{ }; use fuel_core::{ chain_config::{ + random_testnet_wallet, ChainConfig, StateConfig, - TESTNET_WALLET_SECRETS, }, database::Database, service::{ @@ -34,10 +34,7 @@ use fuel_core_storage::{ }; use fuel_core_types::{ fuel_asm::*, - fuel_crypto::{ - coins_bip32::ecdsa::signature::rand_core::SeedableRng, - SecretKey, - }, + fuel_crypto::coins_bip32::ecdsa::signature::rand_core::SeedableRng, fuel_tx::{ consensus_parameters::ConsensusParametersV1, ConsensusParameters, @@ -51,7 +48,6 @@ use rand::Rng; use std::{ iter::repeat, ops::Deref, - str::FromStr, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -73,10 +69,7 @@ fn arb_large_tx( let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); let mut builder = TransactionBuilder::script(script_bytes, vec![]); let asset_id = *builder.get_params().base_asset_id(); - let wallet = SecretKey::from_str( - TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())], - ) - .expect("should parse secret key hex bytes"); + let (wallet, _) = random_testnet_wallet(rng); builder .max_fee_limit(max_fee_limit) .script_gas_limit(22430) @@ -202,10 +195,12 @@ async fn produce_block__lowers_gas_price() { let client = FuelClient::from(srv.bound_address); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + const MAX_FEE: u64 = 189028; + // when let arb_tx_count = 5; for i in 0..arb_tx_count { - let tx = arb_large_tx(189028 + i as Word, &mut rng); + let tx = arb_large_tx(MAX_FEE + i as Word, &mut rng); let _status = client.submit(&tx).await.unwrap(); } // starting gas price From 71226d426d82240eff0408164be4d60c6f1c7e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 1 Nov 2024 13:18:52 +0100 Subject: [PATCH 072/111] Fix typo --- tests/tests/blob.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests/blob.rs b/tests/tests/blob.rs index 813dfea1418..74a1a7d13aa 100644 --- a/tests/tests/blob.rs +++ b/tests/tests/blob.rs @@ -183,7 +183,7 @@ async fn blob__cannot_post_already_existing_blob() { } #[tokio::test] -async fn blob__accessing_nonexitent_blob_panics_vm() { +async fn blob__accessing_nonexistent_blob_panics_vm() { // Given let ctx = TestContext::new().await; let blob_id = BlobId::new([0; 32]); // Nonexistent From 474e207aa82c76e321862faa7a642b6fb824c1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 1 Nov 2024 13:23:38 +0100 Subject: [PATCH 073/111] Fix BLOB integration tests to work with the balance overflow checks --- tests/tests/blob.rs | 122 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 8 deletions(-) diff --git a/tests/tests/blob.rs b/tests/tests/blob.rs index 74a1a7d13aa..cb27a667ed4 100644 --- a/tests/tests/blob.rs +++ b/tests/tests/blob.rs @@ -1,7 +1,12 @@ #![allow(non_snake_case)] +use std::str::FromStr; + use fuel_core::{ - chain_config::StateConfig, + chain_config::{ + random_testnet_wallet, + StateConfig, + }, database::{ database_description::on_chain::OnChain, Database, @@ -22,13 +27,16 @@ use fuel_core_types::{ RegId, }, fuel_tx::{ + Address, BlobBody, BlobId, BlobIdExt, Finalizable, Input, + Output, Transaction, TransactionBuilder, + Word, }, fuel_types::canonical::Serialize, fuel_vm::{ @@ -44,6 +52,11 @@ use fuel_core_types::{ }, }, }; +use rand::{ + rngs::StdRng, + Rng, + SeedableRng, +}; use tokio::io; struct TestContext { @@ -134,14 +147,28 @@ impl TestContext { #[tokio::test] async fn blob__upload_works() { + let mut rng = StdRng::seed_from_u64(2322); + // Given + let (wallet, wallet_str) = random_testnet_wallet(&mut rng); + const SMALL_AMOUNT: Word = 0; let mut ctx = TestContext::new().await; + let utxo_id = rng.gen(); + let input = Input::coin_signed( + utxo_id, + Address::from_str(wallet_str).expect("should parse bytes as address"), + SMALL_AMOUNT, + Default::default(), + Default::default(), + Default::default(), + ); // When let (status, blob_id) = ctx - .new_blob([op::ret(RegId::ONE)].into_iter().collect()) + .new_blob_with_input([op::ret(RegId::ONE)].into_iter().collect(), Some(input)) .await .unwrap(); + assert!(matches!(status, TransactionStatus::Success { .. })); // Then @@ -156,7 +183,18 @@ async fn blob__upload_works() { blob_id.to_bytes(), ) .script_gas_limit(1000000) - .add_fee_input() + .add_unsigned_coin_input( + wallet, + rng.gen(), + SMALL_AMOUNT, + Default::default(), + Default::default(), + ) + .add_output(Output::Change { + amount: SMALL_AMOUNT, + asset_id: Default::default(), + to: Address::from_str(wallet_str).expect("should parse bytes as address"), + }) .finalize_as_transaction(); let tx_status = ctx .client @@ -168,10 +206,27 @@ async fn blob__upload_works() { #[tokio::test] async fn blob__cannot_post_already_existing_blob() { + let mut rng = StdRng::seed_from_u64(2322); + // Given let mut ctx = TestContext::new().await; let payload: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let (status, _blob_id) = ctx.new_blob(payload.clone()).await.unwrap(); + let utxo_id = rng.gen(); + let (_, wallet_str) = random_testnet_wallet(&mut rng); + const SMALL_AMOUNT: Word = 0; + let input = Input::coin_signed( + utxo_id, + Address::from_str(wallet_str).expect("should parse bytes as address"), + SMALL_AMOUNT, + Default::default(), + Default::default(), + Default::default(), + ); + + let (status, _blob_id) = ctx + .new_blob_with_input(payload.clone(), Some(input)) + .await + .unwrap(); assert!(matches!(status, TransactionStatus::Success { .. })); // When @@ -184,7 +239,11 @@ async fn blob__cannot_post_already_existing_blob() { #[tokio::test] async fn blob__accessing_nonexistent_blob_panics_vm() { + let mut rng = StdRng::seed_from_u64(2322); + // Given + let (wallet, wallet_str) = random_testnet_wallet(&mut rng); + const SMALL_AMOUNT: Word = 2; let ctx = TestContext::new().await; let blob_id = BlobId::new([0; 32]); // Nonexistent @@ -200,7 +259,18 @@ async fn blob__accessing_nonexistent_blob_panics_vm() { blob_id.to_bytes(), ) .script_gas_limit(1000000) - .add_fee_input() + .add_unsigned_coin_input( + wallet, + rng.gen(), + SMALL_AMOUNT, + Default::default(), + Default::default(), + ) + .add_output(Output::Change { + amount: SMALL_AMOUNT, + asset_id: Default::default(), + to: Address::from_str(wallet_str).expect("should parse bytes as address"), + }) .finalize_as_transaction(); let tx_status = ctx .client @@ -209,15 +279,35 @@ async fn blob__accessing_nonexistent_blob_panics_vm() { .unwrap(); // Then - assert!(matches!(tx_status, TransactionStatus::Failure { .. })); + assert!( + matches!(tx_status, TransactionStatus::Failure { reason,.. } if reason == "BlobNotFound") + ); } #[tokio::test] async fn blob__can_be_queried_if_uploaded() { + let mut rng = StdRng::seed_from_u64(2322); + // Given let mut ctx = TestContext::new().await; let bytecode: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let (status, blob_id) = ctx.new_blob(bytecode.clone()).await.unwrap(); + let (_, wallet_str) = random_testnet_wallet(&mut rng); + let utxo_id = rng.gen(); + const SMALL_AMOUNT: Word = 0; + let input = Input::coin_signed( + utxo_id, + Address::from_str(wallet_str).expect("should parse bytes as address"), + SMALL_AMOUNT, + Default::default(), + Default::default(), + Default::default(), + ); + + let (status, blob_id) = ctx + .new_blob_with_input(bytecode.clone(), Some(input)) + .await + .unwrap(); + assert!(matches!(status, TransactionStatus::Success { .. })); // When @@ -235,10 +325,26 @@ async fn blob__can_be_queried_if_uploaded() { #[tokio::test] async fn blob__exists_if_uploaded() { + let mut rng = StdRng::seed_from_u64(2322); + // Given let mut ctx = TestContext::new().await; let bytecode: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let (status, blob_id) = ctx.new_blob(bytecode.clone()).await.unwrap(); + let (_, wallet_str) = random_testnet_wallet(&mut rng); + let utxo_id = rng.gen(); + const SMALL_AMOUNT: Word = 0; + let input = Input::coin_signed( + utxo_id, + Address::from_str(wallet_str).expect("should parse bytes as address"), + SMALL_AMOUNT, + Default::default(), + Default::default(), + Default::default(), + ); + let (status, blob_id) = ctx + .new_blob_with_input(bytecode.clone(), Some(input)) + .await + .unwrap(); assert!(matches!(status, TransactionStatus::Success { .. })); // When From ca6057dec800fe09b9033cac19a95d1b5fef7d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Sat, 2 Nov 2024 11:18:08 +0100 Subject: [PATCH 074/111] Revert "Fix BLOB integration tests to work with the balance overflow checks" This reverts commit 474e207aa82c76e321862faa7a642b6fb824c1e6. --- tests/tests/blob.rs | 122 +++----------------------------------------- 1 file changed, 8 insertions(+), 114 deletions(-) diff --git a/tests/tests/blob.rs b/tests/tests/blob.rs index cb27a667ed4..74a1a7d13aa 100644 --- a/tests/tests/blob.rs +++ b/tests/tests/blob.rs @@ -1,12 +1,7 @@ #![allow(non_snake_case)] -use std::str::FromStr; - use fuel_core::{ - chain_config::{ - random_testnet_wallet, - StateConfig, - }, + chain_config::StateConfig, database::{ database_description::on_chain::OnChain, Database, @@ -27,16 +22,13 @@ use fuel_core_types::{ RegId, }, fuel_tx::{ - Address, BlobBody, BlobId, BlobIdExt, Finalizable, Input, - Output, Transaction, TransactionBuilder, - Word, }, fuel_types::canonical::Serialize, fuel_vm::{ @@ -52,11 +44,6 @@ use fuel_core_types::{ }, }, }; -use rand::{ - rngs::StdRng, - Rng, - SeedableRng, -}; use tokio::io; struct TestContext { @@ -147,28 +134,14 @@ impl TestContext { #[tokio::test] async fn blob__upload_works() { - let mut rng = StdRng::seed_from_u64(2322); - // Given - let (wallet, wallet_str) = random_testnet_wallet(&mut rng); - const SMALL_AMOUNT: Word = 0; let mut ctx = TestContext::new().await; - let utxo_id = rng.gen(); - let input = Input::coin_signed( - utxo_id, - Address::from_str(wallet_str).expect("should parse bytes as address"), - SMALL_AMOUNT, - Default::default(), - Default::default(), - Default::default(), - ); // When let (status, blob_id) = ctx - .new_blob_with_input([op::ret(RegId::ONE)].into_iter().collect(), Some(input)) + .new_blob([op::ret(RegId::ONE)].into_iter().collect()) .await .unwrap(); - assert!(matches!(status, TransactionStatus::Success { .. })); // Then @@ -183,18 +156,7 @@ async fn blob__upload_works() { blob_id.to_bytes(), ) .script_gas_limit(1000000) - .add_unsigned_coin_input( - wallet, - rng.gen(), - SMALL_AMOUNT, - Default::default(), - Default::default(), - ) - .add_output(Output::Change { - amount: SMALL_AMOUNT, - asset_id: Default::default(), - to: Address::from_str(wallet_str).expect("should parse bytes as address"), - }) + .add_fee_input() .finalize_as_transaction(); let tx_status = ctx .client @@ -206,27 +168,10 @@ async fn blob__upload_works() { #[tokio::test] async fn blob__cannot_post_already_existing_blob() { - let mut rng = StdRng::seed_from_u64(2322); - // Given let mut ctx = TestContext::new().await; let payload: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let utxo_id = rng.gen(); - let (_, wallet_str) = random_testnet_wallet(&mut rng); - const SMALL_AMOUNT: Word = 0; - let input = Input::coin_signed( - utxo_id, - Address::from_str(wallet_str).expect("should parse bytes as address"), - SMALL_AMOUNT, - Default::default(), - Default::default(), - Default::default(), - ); - - let (status, _blob_id) = ctx - .new_blob_with_input(payload.clone(), Some(input)) - .await - .unwrap(); + let (status, _blob_id) = ctx.new_blob(payload.clone()).await.unwrap(); assert!(matches!(status, TransactionStatus::Success { .. })); // When @@ -239,11 +184,7 @@ async fn blob__cannot_post_already_existing_blob() { #[tokio::test] async fn blob__accessing_nonexistent_blob_panics_vm() { - let mut rng = StdRng::seed_from_u64(2322); - // Given - let (wallet, wallet_str) = random_testnet_wallet(&mut rng); - const SMALL_AMOUNT: Word = 2; let ctx = TestContext::new().await; let blob_id = BlobId::new([0; 32]); // Nonexistent @@ -259,18 +200,7 @@ async fn blob__accessing_nonexistent_blob_panics_vm() { blob_id.to_bytes(), ) .script_gas_limit(1000000) - .add_unsigned_coin_input( - wallet, - rng.gen(), - SMALL_AMOUNT, - Default::default(), - Default::default(), - ) - .add_output(Output::Change { - amount: SMALL_AMOUNT, - asset_id: Default::default(), - to: Address::from_str(wallet_str).expect("should parse bytes as address"), - }) + .add_fee_input() .finalize_as_transaction(); let tx_status = ctx .client @@ -279,35 +209,15 @@ async fn blob__accessing_nonexistent_blob_panics_vm() { .unwrap(); // Then - assert!( - matches!(tx_status, TransactionStatus::Failure { reason,.. } if reason == "BlobNotFound") - ); + assert!(matches!(tx_status, TransactionStatus::Failure { .. })); } #[tokio::test] async fn blob__can_be_queried_if_uploaded() { - let mut rng = StdRng::seed_from_u64(2322); - // Given let mut ctx = TestContext::new().await; let bytecode: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let (_, wallet_str) = random_testnet_wallet(&mut rng); - let utxo_id = rng.gen(); - const SMALL_AMOUNT: Word = 0; - let input = Input::coin_signed( - utxo_id, - Address::from_str(wallet_str).expect("should parse bytes as address"), - SMALL_AMOUNT, - Default::default(), - Default::default(), - Default::default(), - ); - - let (status, blob_id) = ctx - .new_blob_with_input(bytecode.clone(), Some(input)) - .await - .unwrap(); - + let (status, blob_id) = ctx.new_blob(bytecode.clone()).await.unwrap(); assert!(matches!(status, TransactionStatus::Success { .. })); // When @@ -325,26 +235,10 @@ async fn blob__can_be_queried_if_uploaded() { #[tokio::test] async fn blob__exists_if_uploaded() { - let mut rng = StdRng::seed_from_u64(2322); - // Given let mut ctx = TestContext::new().await; let bytecode: Vec = [op::ret(RegId::ONE)].into_iter().collect(); - let (_, wallet_str) = random_testnet_wallet(&mut rng); - let utxo_id = rng.gen(); - const SMALL_AMOUNT: Word = 0; - let input = Input::coin_signed( - utxo_id, - Address::from_str(wallet_str).expect("should parse bytes as address"), - SMALL_AMOUNT, - Default::default(), - Default::default(), - Default::default(), - ); - let (status, blob_id) = ctx - .new_blob_with_input(bytecode.clone(), Some(input)) - .await - .unwrap(); + let (status, blob_id) = ctx.new_blob(bytecode.clone()).await.unwrap(); assert!(matches!(status, TransactionStatus::Success { .. })); // When From 4d430388703747fb180d306281de14643b67422c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Sat, 2 Nov 2024 11:19:02 +0100 Subject: [PATCH 075/111] Revert "Fix more integration tests to work with the balance overflow checks" This reverts commit de11071b8dbafb8b6d40048eaf847f5adc28078f. --- crates/chain-config/src/config/state.rs | 17 ----------------- tests/test-helpers/src/lib.rs | 23 +++++++++-------------- tests/tests/blocks.rs | 2 +- tests/tests/gas_price.rs | 17 +++++++++++------ 4 files changed, 21 insertions(+), 38 deletions(-) diff --git a/crates/chain-config/src/config/state.rs b/crates/chain-config/src/config/state.rs index f64954dbab3..c8fee0468fd 100644 --- a/crates/chain-config/src/config/state.rs +++ b/crates/chain-config/src/config/state.rs @@ -37,12 +37,6 @@ use fuel_core_types::{ fuel_vm::BlobData, }; use itertools::Itertools; -#[cfg(feature = "test-helpers")] -use rand::{ - CryptoRng, - Rng, - RngCore, -}; use serde::{ Deserialize, Serialize, @@ -93,17 +87,6 @@ pub const TESTNET_WALLET_SECRETS: [&str; 5] = [ "0x7f8a325504e7315eda997db7861c9447f5c3eff26333b20180475d94443a10c6", ]; -#[cfg(feature = "test-helpers")] -pub fn random_testnet_wallet( - rng: &mut (impl CryptoRng + RngCore), -) -> (SecretKey, &'static str) { - let wallet_str = - TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())]; - let wallet = - SecretKey::from_str(wallet_str).expect("should parse secret key hex bytes"); - (wallet, wallet_str) -} - #[derive(Default, Copy, Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct LastBlockConfig { /// The block height of the last block. diff --git a/tests/test-helpers/src/lib.rs b/tests/test-helpers/src/lib.rs index 32f07871fde..89e3f0c0616 100644 --- a/tests/test-helpers/src/lib.rs +++ b/tests/test-helpers/src/lib.rs @@ -1,6 +1,3 @@ -use std::str::FromStr; - -use fuel_core::chain_config::random_testnet_wallet; use fuel_core_client::client::{ types::TransactionStatus, FuelClient, @@ -12,11 +9,9 @@ use fuel_core_types::{ }, fuel_crypto::SecretKey, fuel_tx::{ - Address, Output, Transaction, TransactionBuilder, - Word, }, }; use rand::{ @@ -38,27 +33,27 @@ pub async fn send_graph_ql_query(url: &str, query: &str) -> String { response.text().await.unwrap() } -pub fn make_tx(rng: &mut (impl CryptoRng + RngCore), max_gas_limit: u64) -> Transaction { - const SMALL_AMOUNT: Word = 2; - - let (wallet, wallet_str) = random_testnet_wallet(rng); - +pub fn make_tx( + rng: &mut (impl CryptoRng + RngCore), + i: u64, + max_gas_limit: u64, +) -> Transaction { TransactionBuilder::script( op::ret(RegId::ONE).to_bytes().into_iter().collect(), vec![], ) .script_gas_limit(max_gas_limit / 2) .add_unsigned_coin_input( - wallet, + SecretKey::random(rng), rng.gen(), - SMALL_AMOUNT, + 1000 + i, Default::default(), Default::default(), ) .add_output(Output::Change { - amount: SMALL_AMOUNT, + amount: 0, asset_id: Default::default(), - to: Address::from_str(wallet_str).expect("should parse bytes as address"), + to: rng.gen(), }) .finalize_as_transaction() } diff --git a/tests/tests/blocks.rs b/tests/tests/blocks.rs index 2fb0e99bb6a..7a5ba4688d5 100644 --- a/tests/tests/blocks.rs +++ b/tests/tests/blocks.rs @@ -481,7 +481,7 @@ mod full_block { let tx_count: u64 = 66_000; let txs = (1..=tx_count) - .map(|_| test_helpers::make_tx(&mut rng, max_gas_limit)) + .map(|i| test_helpers::make_tx(&mut rng, i, max_gas_limit)) .collect_vec(); // When diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index ed0ce66b810..d92214969a9 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -6,9 +6,9 @@ use crate::helpers::{ }; use fuel_core::{ chain_config::{ - random_testnet_wallet, ChainConfig, StateConfig, + TESTNET_WALLET_SECRETS, }, database::Database, service::{ @@ -34,7 +34,10 @@ use fuel_core_storage::{ }; use fuel_core_types::{ fuel_asm::*, - fuel_crypto::coins_bip32::ecdsa::signature::rand_core::SeedableRng, + fuel_crypto::{ + coins_bip32::ecdsa::signature::rand_core::SeedableRng, + SecretKey, + }, fuel_tx::{ consensus_parameters::ConsensusParametersV1, ConsensusParameters, @@ -48,6 +51,7 @@ use rand::Rng; use std::{ iter::repeat, ops::Deref, + str::FromStr, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -69,7 +73,10 @@ fn arb_large_tx( let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); let mut builder = TransactionBuilder::script(script_bytes, vec![]); let asset_id = *builder.get_params().base_asset_id(); - let (wallet, _) = random_testnet_wallet(rng); + let wallet = SecretKey::from_str( + TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())], + ) + .expect("should parse secret key hex bytes"); builder .max_fee_limit(max_fee_limit) .script_gas_limit(22430) @@ -195,12 +202,10 @@ async fn produce_block__lowers_gas_price() { let client = FuelClient::from(srv.bound_address); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); - const MAX_FEE: u64 = 189028; - // when let arb_tx_count = 5; for i in 0..arb_tx_count { - let tx = arb_large_tx(MAX_FEE + i as Word, &mut rng); + let tx = arb_large_tx(189028 + i as Word, &mut rng); let _status = client.submit(&tx).await.unwrap(); } // starting gas price From e3d898db38b273447db44f8a5d20381160a62dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Sat, 2 Nov 2024 11:19:24 +0100 Subject: [PATCH 076/111] Revert "Fix `produce_block__raises_gas_price` after balance overflow check was added" This reverts commit 895d9da0928d3c799ae292d50be1853059cfc359. --- tests/tests/gas_price.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index d92214969a9..18507785eec 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -8,7 +8,6 @@ use fuel_core::{ chain_config::{ ChainConfig, StateConfig, - TESTNET_WALLET_SECRETS, }, database::Database, service::{ @@ -51,7 +50,6 @@ use rand::Rng; use std::{ iter::repeat, ops::Deref, - str::FromStr, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -73,17 +71,13 @@ fn arb_large_tx( let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); let mut builder = TransactionBuilder::script(script_bytes, vec![]); let asset_id = *builder.get_params().base_asset_id(); - let wallet = SecretKey::from_str( - TESTNET_WALLET_SECRETS[rng.gen_range(0..TESTNET_WALLET_SECRETS.len())], - ) - .expect("should parse secret key hex bytes"); builder .max_fee_limit(max_fee_limit) .script_gas_limit(22430) .add_unsigned_coin_input( - wallet, + SecretKey::random(rng), rng.gen(), - max_fee_limit, + u32::MAX as u64, asset_id, Default::default(), ) @@ -155,12 +149,10 @@ async fn produce_block__raises_gas_price() { let client = FuelClient::from(srv.bound_address); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); - const MAX_FEE: u64 = 189028; - // when let arb_tx_count = 10; for i in 0..arb_tx_count { - let tx = arb_large_tx(MAX_FEE + i as Word, &mut rng); + let tx = arb_large_tx(189028 + i as Word, &mut rng); let _status = client.submit(&tx).await.unwrap(); } // starting gas price From 9848f64c844942274f2657c57dbb594c092ab449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Sat, 2 Nov 2024 11:41:17 +0100 Subject: [PATCH 077/111] Do not bail when balances cannot be updated, log the error instead --- crates/fuel-core/src/graphql_api/worker_service.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 5829d12ec8f..6c1b31f7c13 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -324,7 +324,15 @@ where T: OffChainDatabaseTransaction, { for event in events { - process_balances_update(event.deref(), block_st_transaction, balances_enabled)?; + if let Err(err) = + process_balances_update(event.deref(), block_st_transaction, balances_enabled) + { + // TODO[RC]: This means that we were not able to update the balances, most likely due to overflow. + // This is a fatal error, because the balances are not consistent with the actual state of the chain. + // However, if we bail here, a lot of integration tests will start failing, because they often + // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. + tracing::error!(%err, "Processing balances") + } match event.deref() { Event::MessageImported(message) => { block_st_transaction From dae0af505de5664dfa0a635f5c56f465ca90a3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 08:17:18 +0100 Subject: [PATCH 078/111] Total balance of assets is now represented as `u128` --- crates/fuel-core/src/graphql_api/ports.rs | 6 +- .../src/graphql_api/storage/balances.rs | 7 +- .../src/graphql_api/worker_service.rs | 86 ++++++++++++------- crates/fuel-core/src/query/balance.rs | 12 ++- crates/fuel-core/src/query/contract.rs | 4 +- crates/fuel-core/src/schema/balance.rs | 4 +- crates/fuel-core/src/schema/contract.rs | 4 +- crates/fuel-core/src/schema/scalars.rs | 1 + .../service/adapters/graphql_api/off_chain.rs | 23 ++--- .../service/adapters/graphql_api/on_chain.rs | 2 +- crates/types/src/services/graphql_api.rs | 2 +- 11 files changed, 91 insertions(+), 60 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index b87b37603f0..50d14a1f5d5 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -67,6 +67,8 @@ use std::{ sync::Arc, }; +use super::storage::balances::TotalBalanceAmount; + pub trait OffChainDatabase: Send + Sync { fn block_height(&self, block_id: &BlockId) -> StorageResult; @@ -79,13 +81,13 @@ pub trait OffChainDatabase: Send + Sync { owner: &Address, asset_id: &AssetId, base_asset_id: &AssetId, - ) -> StorageResult; + ) -> StorageResult; fn balances( &self, owner: &Address, base_asset_id: &AssetId, - ) -> StorageResult>; + ) -> StorageResult>; fn owned_coins_ids( &self, diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 8415964521a..10aec2862e7 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -20,7 +20,8 @@ use rand::{ Rng, }; -pub type Amount = u64; +pub type ItemAmount = u64; +pub type TotalBalanceAmount = u128; double_key!(BalancesKey, Address, address, AssetId, asset_id); impl Distribution for Standard { @@ -43,7 +44,7 @@ pub struct CoinBalances; impl Mappable for CoinBalances { type Key = BalancesKey; type OwnedKey = Self::Key; - type Value = Amount; + type Value = TotalBalanceAmount; type OwnedValue = Self::Value; } @@ -62,7 +63,7 @@ pub struct MessageBalances; impl Mappable for MessageBalances { type Key = Address; type OwnedKey = Self::Key; - type Value = Amount; + type Value = TotalBalanceAmount; type OwnedValue = Self::Value; } diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 6c1b31f7c13..bbcd2b76313 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -2,8 +2,9 @@ use super::{ da_compression::da_compress_block, storage::{ balances::{ - Amount, BalancesKey, + ItemAmount, + TotalBalanceAmount, }, old::{ OldFuelBlockConsensus, @@ -209,44 +210,51 @@ trait DatabaseItemWithAmount { type Storage: Mappable; fn key(&self) -> ::Key; - fn amount(&self) -> Amount; + fn amount(&self) -> ItemAmount; } -impl DatabaseItemWithAmount for Coin { +impl DatabaseItemWithAmount for &Coin { type Storage = CoinBalances; fn key(&self) -> ::Key { BalancesKey::new(&self.owner, &self.asset_id) } - fn amount(&self) -> Amount { - self.amount + fn amount(&self) -> ItemAmount { + self.amount.into() } } -impl DatabaseItemWithAmount for Message { +impl DatabaseItemWithAmount for &Message { type Storage = MessageBalances; fn key(&self) -> ::Key { *self.recipient() } - fn amount(&self) -> Amount { - Self::amount(self) + fn amount(&self) -> ItemAmount { + (**self).amount().into() } } trait BalanceIndexationUpdater: DatabaseItemWithAmount { - fn update_balances(&self, tx: &mut T, updater: F) -> StorageResult<()> + type TotalBalance: From<::OwnedValue> + core::fmt::Display; + + fn update_balances( + &self, + tx: &mut T, + updater: UpdaterFn, + ) -> StorageResult<()> where ::Key: Sized + core::fmt::Display, ::Value: Sized + core::fmt::Display, ::OwnedValue: Default + core::fmt::Display, + UpdaterFn: Fn(Self::TotalBalance, ItemAmount) -> Option, T: OffChainDatabaseTransaction + StorageMutate, - F: Fn( - Cow<::OwnedValue>, - Amount, - ) -> Option<::Value>, + <::Storage as Mappable>::Value: + std::convert::From, + <::Storage as Mappable>::Value: + std::convert::From<::TotalBalance>, fuel_core_storage::Error: From<>::Error>, { let key = self.key(); @@ -254,7 +262,7 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { let storage = tx.storage::(); let current_balance = storage.get(&key)?.unwrap_or_default(); let prev_balance = current_balance.clone(); - match updater(current_balance, amount) { + match updater(current_balance.as_ref().clone().into(), amount.into()) { Some(new_balance) => { debug!( %key, @@ -264,7 +272,7 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { "changing balance"); let storage = tx.storage::(); - Ok(storage.insert(&key, &new_balance)?) + Ok(storage.insert(&key, &new_balance.into())?) } None => { error!( @@ -278,8 +286,12 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { } } -impl BalanceIndexationUpdater for Coin {} -impl BalanceIndexationUpdater for Message {} +impl BalanceIndexationUpdater for &Coin { + type TotalBalance = TotalBalanceAmount; +} +impl BalanceIndexationUpdater for &Message { + type TotalBalance = TotalBalanceAmount; +} fn process_balances_update( event: &Event, @@ -293,22 +305,30 @@ where return Ok(()); } match event { - Event::MessageImported(message) => message - .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.checked_add(amount) - }), - Event::MessageConsumed(message) => message - .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.checked_sub(amount) - }), - Event::CoinCreated(coin) => coin - .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.checked_add(amount) - }), - Event::CoinConsumed(coin) => coin - .update_balances(block_st_transaction, |balance: Cow, amount| { - balance.checked_sub(amount) - }), + Event::MessageImported(message) => message.update_balances( + block_st_transaction, + |balance: TotalBalanceAmount, amount: ItemAmount| { + balance.checked_add(amount as TotalBalanceAmount) + }, + ), + Event::MessageConsumed(message) => message.update_balances( + block_st_transaction, + |balance: TotalBalanceAmount, amount: ItemAmount| { + balance.checked_sub(amount as TotalBalanceAmount) + }, + ), + Event::CoinCreated(coin) => coin.update_balances( + block_st_transaction, + |balance: TotalBalanceAmount, amount: ItemAmount| { + balance.checked_add(amount as TotalBalanceAmount) + }, + ), + Event::CoinConsumed(coin) => coin.update_balances( + block_st_transaction, + |balance: TotalBalanceAmount, amount: ItemAmount| { + balance.checked_sub(amount as TotalBalanceAmount) + }, + ), Event::ForcedTransactionFailed { .. } => Ok(()), } } diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 2f4204ab2ce..79c6c6a2c87 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -4,7 +4,10 @@ use std::{ future, }; -use crate::fuel_core_graphql_api::database::ReadView; +use crate::{ + fuel_core_graphql_api::database::ReadView, + graphql_api::storage::balances::TotalBalanceAmount, +}; use asset_query::{ AssetQuery, AssetSpendTarget, @@ -60,7 +63,7 @@ impl ReadView { Ok(balance.saturating_add(amount)) } }) - .await? + .await? as TotalBalanceAmount }; Ok(AddressBalance { @@ -105,10 +108,11 @@ impl ReadView { .try_fold( HashMap::new(), move |mut amounts_per_asset, coin| async move { - let amount: &mut u64 = amounts_per_asset + let amount: &mut TotalBalanceAmount = amounts_per_asset .entry(*coin.asset_id(base_asset_id)) .or_default(); - *amount = amount.saturating_add(coin.amount()); + // TODO[RC]: checked_add + *amount = amount.saturating_add(coin.amount() as TotalBalanceAmount); Ok(amounts_per_asset) }, ) diff --git a/crates/fuel-core/src/query/contract.rs b/crates/fuel-core/src/query/contract.rs index fa75e05a874..6c30d60e239 100644 --- a/crates/fuel-core/src/query/contract.rs +++ b/crates/fuel-core/src/query/contract.rs @@ -1,4 +1,4 @@ -use crate::fuel_core_graphql_api::database::ReadView; +use crate::{fuel_core_graphql_api::database::ReadView, graphql_api::storage::balances::TotalBalanceAmount}; use fuel_core_storage::{ not_found, tables::{ @@ -47,7 +47,7 @@ impl ReadView { .storage::() .get(&(&contract_id, &asset_id).into())? .ok_or(not_found!(ContractsAssets))? - .into_owned(); + .into_owned() as TotalBalanceAmount; Ok(ContractBalance { owner: contract_id, diff --git a/crates/fuel-core/src/schema/balance.rs b/crates/fuel-core/src/schema/balance.rs index 140bb81256f..b6b95228e83 100644 --- a/crates/fuel-core/src/schema/balance.rs +++ b/crates/fuel-core/src/schema/balance.rs @@ -7,7 +7,7 @@ use crate::{ scalars::{ Address, AssetId, - U64, + U128, }, ReadViewProvider, }, @@ -33,7 +33,7 @@ impl Balance { self.0.owner.into() } - async fn amount(&self) -> U64 { + async fn amount(&self) -> U128 { self.0.amount.into() } diff --git a/crates/fuel-core/src/schema/contract.rs b/crates/fuel-core/src/schema/contract.rs index 88e43c98a3d..943ba89207e 100644 --- a/crates/fuel-core/src/schema/contract.rs +++ b/crates/fuel-core/src/schema/contract.rs @@ -9,7 +9,7 @@ use crate::{ ContractId, HexString, Salt, - U64, + U128, }, ReadViewProvider, }, @@ -99,7 +99,7 @@ impl ContractBalance { self.0.owner.into() } - async fn amount(&self) -> U64 { + async fn amount(&self) -> U128 { self.0.amount.into() } diff --git a/crates/fuel-core/src/schema/scalars.rs b/crates/fuel-core/src/schema/scalars.rs index e75b3271f98..d3ddb7df20b 100644 --- a/crates/fuel-core/src/schema/scalars.rs +++ b/crates/fuel-core/src/schema/scalars.rs @@ -79,6 +79,7 @@ macro_rules! number_scalar { }; } +number_scalar!(U128, u128, "U128"); number_scalar!(U64, u64, "U64"); number_scalar!(U32, u32, "U32"); number_scalar!(U16, u16, "U16"); diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 3329935802f..a0bb450550d 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -25,7 +25,7 @@ use crate::{ balances::{ BalancesKey, CoinBalances, - MessageBalances, + MessageBalances, TotalBalanceAmount, }, old::{ OldFuelBlockConsensus, @@ -207,19 +207,21 @@ impl OffChainDatabase for OffChainIterableKeyValueView { owner: &Address, asset_id: &AssetId, base_asset_id: &AssetId, - ) -> StorageResult { + ) -> StorageResult { let coins = self .storage_as_ref::() .get(&BalancesKey::new(owner, asset_id))? - .unwrap_or_default(); + .unwrap_or_default() + .into_owned() as TotalBalanceAmount; if base_asset_id == asset_id { let messages = self .storage_as_ref::() .get(owner)? - .unwrap_or_default(); + .unwrap_or_default() + .into_owned() as TotalBalanceAmount; - let total = coins.checked_add(*messages).ok_or(anyhow::anyhow!( + let total = coins.checked_add(messages).ok_or(anyhow::anyhow!( "Total balance overflow: coins: {coins}, messages: {messages}" ))?; @@ -227,7 +229,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { Ok(total) } else { debug!(%coins, "total balance"); - Ok(*coins) + Ok(coins) } } @@ -235,17 +237,17 @@ impl OffChainDatabase for OffChainIterableKeyValueView { &self, owner: &Address, base_asset_id: &AssetId, - ) -> StorageResult> { + ) -> StorageResult> { let mut balances = BTreeMap::new(); for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { let key = balance_key?; let asset_id = key.asset_id(); let messages = if base_asset_id == asset_id { - *self - .storage_as_ref::() + self.storage_as_ref::() .get(owner)? .unwrap_or_default() + .into_owned() as TotalBalanceAmount } else { 0 }; @@ -253,7 +255,8 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let coins = self .storage_as_ref::() .get(&key)? - .unwrap_or_default(); + .unwrap_or_default() + .into_owned() as TotalBalanceAmount; let total = coins.checked_add(messages).ok_or(anyhow::anyhow!( "Total balance overflow: coins: {coins}, messages: {messages}" diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index b3a6d860e76..bca1efc03fa 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -118,7 +118,7 @@ impl DatabaseContracts for OnChainIterableKeyValueView { self.filter_contract_balances(contract, start_asset, Some(direction)) .map_ok(|entry| ContractBalance { owner: *entry.key.contract_id(), - amount: entry.value, + amount: entry.value as u128, asset_id: *entry.key.asset_id(), }) .map(|res| res.map_err(StorageError::from)) diff --git a/crates/types/src/services/graphql_api.rs b/crates/types/src/services/graphql_api.rs index b38d73e0e03..922f1b2db7e 100644 --- a/crates/types/src/services/graphql_api.rs +++ b/crates/types/src/services/graphql_api.rs @@ -11,7 +11,7 @@ pub struct Balance { /// Owner of the asset. pub owner: Owner, /// The cumulative amount of the asset. - pub amount: u64, + pub amount: u128, /// The identifier of the asset. pub asset_id: AssetId, } From facbd2ef66acf3617185e9609e535f3fa9128fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 12:16:46 +0100 Subject: [PATCH 079/111] Infer types in `process_balances_update()` --- .../src/graphql_api/worker_service.rs | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index bbcd2b76313..ea8c705f4b2 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -305,30 +305,22 @@ where return Ok(()); } match event { - Event::MessageImported(message) => message.update_balances( - block_st_transaction, - |balance: TotalBalanceAmount, amount: ItemAmount| { + Event::MessageImported(message) => message + .update_balances(block_st_transaction, |balance, amount| { balance.checked_add(amount as TotalBalanceAmount) - }, - ), - Event::MessageConsumed(message) => message.update_balances( - block_st_transaction, - |balance: TotalBalanceAmount, amount: ItemAmount| { + }), + Event::MessageConsumed(message) => message + .update_balances(block_st_transaction, |balance, amount| { balance.checked_sub(amount as TotalBalanceAmount) - }, - ), - Event::CoinCreated(coin) => coin.update_balances( - block_st_transaction, - |balance: TotalBalanceAmount, amount: ItemAmount| { + }), + Event::CoinCreated(coin) => coin + .update_balances(block_st_transaction, |balance, amount| { balance.checked_add(amount as TotalBalanceAmount) - }, - ), - Event::CoinConsumed(coin) => coin.update_balances( - block_st_transaction, - |balance: TotalBalanceAmount, amount: ItemAmount| { + }), + Event::CoinConsumed(coin) => coin + .update_balances(block_st_transaction, |balance, amount| { balance.checked_sub(amount as TotalBalanceAmount) - }, - ), + }), Event::ForcedTransactionFailed { .. } => Ok(()), } } From 9b8e4913fa6678a710a6d760574dfb882b49300f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 12:19:45 +0100 Subject: [PATCH 080/111] Simplify trait bounds in `update_balances()` --- crates/fuel-core/src/graphql_api/worker_service.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index ea8c705f4b2..298eea4338d 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -247,14 +247,11 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { ) -> StorageResult<()> where ::Key: Sized + core::fmt::Display, - ::Value: Sized + core::fmt::Display, ::OwnedValue: Default + core::fmt::Display, UpdaterFn: Fn(Self::TotalBalance, ItemAmount) -> Option, T: OffChainDatabaseTransaction + StorageMutate, - <::Storage as Mappable>::Value: - std::convert::From, - <::Storage as Mappable>::Value: - std::convert::From<::TotalBalance>, + ::Value: + From<::TotalBalance>, fuel_core_storage::Error: From<>::Error>, { let key = self.key(); From 54087fcabb39bbf23c80c1fbf6af87861f60bb2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 12:20:46 +0100 Subject: [PATCH 081/111] Fix formatting --- crates/fuel-core/src/query/contract.rs | 5 ++++- .../fuel-core/src/service/adapters/graphql_api/off_chain.rs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/query/contract.rs b/crates/fuel-core/src/query/contract.rs index 6c30d60e239..bc9e11b31b7 100644 --- a/crates/fuel-core/src/query/contract.rs +++ b/crates/fuel-core/src/query/contract.rs @@ -1,4 +1,7 @@ -use crate::{fuel_core_graphql_api::database::ReadView, graphql_api::storage::balances::TotalBalanceAmount}; +use crate::{ + fuel_core_graphql_api::database::ReadView, + graphql_api::storage::balances::TotalBalanceAmount, +}; use fuel_core_storage::{ not_found, tables::{ diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index a0bb450550d..67bb2741369 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -25,7 +25,8 @@ use crate::{ balances::{ BalancesKey, CoinBalances, - MessageBalances, TotalBalanceAmount, + MessageBalances, + TotalBalanceAmount, }, old::{ OldFuelBlockConsensus, From 20081fa3140080d0533ae188e774c55dbcc064f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 14:36:12 +0100 Subject: [PATCH 082/111] Satisfy Clippy --- crates/fuel-core/src/graphql_api/worker_service.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 298eea4338d..68589dfc8a1 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -221,7 +221,7 @@ impl DatabaseItemWithAmount for &Coin { } fn amount(&self) -> ItemAmount { - self.amount.into() + self.amount } } @@ -233,7 +233,7 @@ impl DatabaseItemWithAmount for &Message { } fn amount(&self) -> ItemAmount { - (**self).amount().into() + (**self).amount() } } @@ -259,7 +259,7 @@ trait BalanceIndexationUpdater: DatabaseItemWithAmount { let storage = tx.storage::(); let current_balance = storage.get(&key)?.unwrap_or_default(); let prev_balance = current_balance.clone(); - match updater(current_balance.as_ref().clone().into(), amount.into()) { + match updater(current_balance.as_ref().clone().into(), amount) { Some(new_balance) => { debug!( %key, From 8b9f4e82cd08f5d3890cf049004cbe5ce7988466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 6 Nov 2024 14:36:40 +0100 Subject: [PATCH 083/111] Update tests --- crates/fuel-core/src/executor.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 412d13c18cf..de656f64550 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -548,12 +548,16 @@ mod tests { .unwrap() .unwrap(); assert_eq!(asset_id, AssetId::zeroed()); - assert_eq!(amount, expected_fee_amount_1); + assert_eq!(amount, expected_fee_amount_1 as u128); + + let amount_u64: u64 = amount + .try_into() + .expect("amount should be lower than u64::MAX"); let script = TxBuilder::new(2u64) .script_gas_limit(limit) - .max_fee_limit(amount) - .coin_input(AssetId::BASE, amount) + .max_fee_limit(amount_u64) + .coin_input(AssetId::BASE, amount_u64) .change_output(AssetId::BASE) .build() .transaction() @@ -638,8 +642,13 @@ mod tests { .next() .unwrap() .unwrap(); + + let amount_u64: u64 = amount + .try_into() + .expect("amount should be lower than u64::MAX"); + assert_eq!(asset_id, AssetId::zeroed()); - assert_eq!(amount, expected_fee_amount_1 + expected_fee_amount_2); + assert_eq!(amount_u64, expected_fee_amount_1 + expected_fee_amount_2); } #[test] From e819e23a26cbfe3b426dfe3a7eeef0ce688596a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 7 Nov 2024 05:31:07 +0100 Subject: [PATCH 084/111] Asset balance queries now return U128 instead of U64. --- CHANGELOG.md | 3 +++ bin/e2e-test-client/src/test_context.rs | 2 +- crates/client/assets/schema.sdl | 4 +++- crates/client/src/client.rs | 2 +- crates/client/src/client/schema/balance.rs | 4 ++-- crates/client/src/client/schema/primitives.rs | 1 + crates/client/src/client/types/balance.rs | 2 +- crates/fuel-core/src/executor.rs | 16 ++++------------ crates/fuel-core/src/query/contract.rs | 7 ++----- crates/fuel-core/src/schema/contract.rs | 4 ++-- .../src/service/adapters/graphql_api/on_chain.rs | 2 +- crates/types/src/services/graphql_api.rs | 8 ++++---- tests/tests/chain.rs | 4 ++-- tests/tests/fee_collection_contract.rs | 2 +- 14 files changed, 28 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 053d3258a87..293b1aa9e04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed - [2378](https://github.com/FuelLabs/fuel-core/pull/2378): Use cached hash of the topic instead of calculating it on each publishing gossip message. +#### Breaking +- [2383](https://github.com/FuelLabs/fuel-core/pull/2383): Asset balance queries now return U128 instead of U64. + ## [Version 0.40.0] ### Added diff --git a/bin/e2e-test-client/src/test_context.rs b/bin/e2e-test-client/src/test_context.rs index 1cb6bcb8b07..d5e98c121d4 100644 --- a/bin/e2e-test-client/src/test_context.rs +++ b/bin/e2e-test-client/src/test_context.rs @@ -99,7 +99,7 @@ impl Wallet { } /// returns the balance associated with a wallet - pub async fn balance(&self, asset_id: Option) -> anyhow::Result { + pub async fn balance(&self, asset_id: Option) -> anyhow::Result { self.client .balance(&self.address, Some(&asset_id.unwrap_or_default())) .await diff --git a/crates/client/assets/schema.sdl b/crates/client/assets/schema.sdl index b9048362caa..42aea952a82 100644 --- a/crates/client/assets/schema.sdl +++ b/crates/client/assets/schema.sdl @@ -4,7 +4,7 @@ scalar AssetId type Balance { owner: Address! - amount: U64! + amount: U128! assetId: AssetId! } @@ -1259,6 +1259,8 @@ enum TxParametersVersion { scalar TxPointer +scalar U128 + scalar U16 scalar U32 diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index 65789f6c9c3..ed032d5888e 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -1054,7 +1054,7 @@ impl FuelClient { &self, owner: &Address, asset_id: Option<&AssetId>, - ) -> io::Result { + ) -> io::Result { let owner: schema::Address = (*owner).into(); let asset_id: schema::AssetId = match asset_id { Some(asset_id) => (*asset_id).into(), diff --git a/crates/client/src/client/schema/balance.rs b/crates/client/src/client/schema/balance.rs index 89c5dbca32d..c20da989f62 100644 --- a/crates/client/src/client/schema/balance.rs +++ b/crates/client/src/client/schema/balance.rs @@ -4,7 +4,7 @@ use crate::client::{ Address, AssetId, PageInfo, - U64, + U128, }, PageDirection, PaginationRequest, @@ -99,7 +99,7 @@ pub struct BalanceEdge { #[cynic(schema_path = "./assets/schema.sdl")] pub struct Balance { pub owner: Address, - pub amount: U64, + pub amount: U128, pub asset_id: AssetId, } diff --git a/crates/client/src/client/schema/primitives.rs b/crates/client/src/client/schema/primitives.rs index 1559c835844..4c2852f852a 100644 --- a/crates/client/src/client/schema/primitives.rs +++ b/crates/client/src/client/schema/primitives.rs @@ -272,6 +272,7 @@ macro_rules! number_scalar { }; } +number_scalar!(U128, u128); number_scalar!(U64, u64); number_scalar!(U32, u32); number_scalar!(U16, u16); diff --git a/crates/client/src/client/types/balance.rs b/crates/client/src/client/types/balance.rs index 334fc5dec46..3220d9c036c 100644 --- a/crates/client/src/client/types/balance.rs +++ b/crates/client/src/client/types/balance.rs @@ -10,7 +10,7 @@ use crate::client::{ #[derive(Clone, Copy, Debug, PartialEq)] pub struct Balance { pub owner: Address, - pub amount: u64, + pub amount: u128, pub asset_id: AssetId, } diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index de656f64550..05965d5d71c 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -548,16 +548,12 @@ mod tests { .unwrap() .unwrap(); assert_eq!(asset_id, AssetId::zeroed()); - assert_eq!(amount, expected_fee_amount_1 as u128); - - let amount_u64: u64 = amount - .try_into() - .expect("amount should be lower than u64::MAX"); + assert_eq!(amount, expected_fee_amount_1); let script = TxBuilder::new(2u64) .script_gas_limit(limit) - .max_fee_limit(amount_u64) - .coin_input(AssetId::BASE, amount_u64) + .max_fee_limit(amount) + .coin_input(AssetId::BASE, amount) .change_output(AssetId::BASE) .build() .transaction() @@ -643,12 +639,8 @@ mod tests { .unwrap() .unwrap(); - let amount_u64: u64 = amount - .try_into() - .expect("amount should be lower than u64::MAX"); - assert_eq!(asset_id, AssetId::zeroed()); - assert_eq!(amount_u64, expected_fee_amount_1 + expected_fee_amount_2); + assert_eq!(amount, expected_fee_amount_1 + expected_fee_amount_2); } #[test] diff --git a/crates/fuel-core/src/query/contract.rs b/crates/fuel-core/src/query/contract.rs index bc9e11b31b7..fa75e05a874 100644 --- a/crates/fuel-core/src/query/contract.rs +++ b/crates/fuel-core/src/query/contract.rs @@ -1,7 +1,4 @@ -use crate::{ - fuel_core_graphql_api::database::ReadView, - graphql_api::storage::balances::TotalBalanceAmount, -}; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ not_found, tables::{ @@ -50,7 +47,7 @@ impl ReadView { .storage::() .get(&(&contract_id, &asset_id).into())? .ok_or(not_found!(ContractsAssets))? - .into_owned() as TotalBalanceAmount; + .into_owned(); Ok(ContractBalance { owner: contract_id, diff --git a/crates/fuel-core/src/schema/contract.rs b/crates/fuel-core/src/schema/contract.rs index 943ba89207e..88e43c98a3d 100644 --- a/crates/fuel-core/src/schema/contract.rs +++ b/crates/fuel-core/src/schema/contract.rs @@ -9,7 +9,7 @@ use crate::{ ContractId, HexString, Salt, - U128, + U64, }, ReadViewProvider, }, @@ -99,7 +99,7 @@ impl ContractBalance { self.0.owner.into() } - async fn amount(&self) -> U128 { + async fn amount(&self) -> U64 { self.0.amount.into() } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index bca1efc03fa..b3a6d860e76 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -118,7 +118,7 @@ impl DatabaseContracts for OnChainIterableKeyValueView { self.filter_contract_balances(contract, start_asset, Some(direction)) .map_ok(|entry| ContractBalance { owner: *entry.key.contract_id(), - amount: entry.value as u128, + amount: entry.value, asset_id: *entry.key.asset_id(), }) .map(|res| res.map_err(StorageError::from)) diff --git a/crates/types/src/services/graphql_api.rs b/crates/types/src/services/graphql_api.rs index 922f1b2db7e..efcfdc99ecd 100644 --- a/crates/types/src/services/graphql_api.rs +++ b/crates/types/src/services/graphql_api.rs @@ -7,17 +7,17 @@ use crate::fuel_types::{ }; /// The cumulative balance(`amount`) of the `Owner` of `asset_id`. -pub struct Balance { +pub struct Balance { /// Owner of the asset. pub owner: Owner, /// The cumulative amount of the asset. - pub amount: u128, + pub amount: Amount, /// The identifier of the asset. pub asset_id: AssetId, } /// The alias for the `Balance` of the address. -pub type AddressBalance = Balance
; +pub type AddressBalance = Balance; /// The alias for the `Balance` of the contract. -pub type ContractBalance = Balance; +pub type ContractBalance = Balance; diff --git a/tests/tests/chain.rs b/tests/tests/chain.rs index c5c62b8f600..43470f7c0d3 100644 --- a/tests/tests/chain.rs +++ b/tests/tests/chain.rs @@ -170,11 +170,11 @@ async fn network_operates_with_non_zero_base_asset_id() { .expect("transaction should insert"); // Then - let expected_fee = 1; + let expected_fee = 1_u128; assert!(matches!(result, TransactionStatus::Success { .. })); let balance = client .balance(&owner, Some(&new_base_asset_id)) .await .expect("Should fetch the balance"); - assert_eq!(balance, amount - expected_fee); + assert_eq!(balance, amount as u128 - expected_fee); } diff --git a/tests/tests/fee_collection_contract.rs b/tests/tests/fee_collection_contract.rs index 54426b5c293..e449e6fa17b 100644 --- a/tests/tests/fee_collection_contract.rs +++ b/tests/tests/fee_collection_contract.rs @@ -227,7 +227,7 @@ async fn happy_path() { // Make sure that the full balance was been withdrawn assert_eq!( ctx.client.balance(&ctx.address, None).await.unwrap(), - contract_balance_before_collect + contract_balance_before_collect as u128 ); } From de1609936764a7498f7f6d8c0b9eb2da729fc220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 8 Nov 2024 10:25:17 +0100 Subject: [PATCH 085/111] Log errors when balance cannot be calculated in the legacy calculation flow (without indexation) --- crates/fuel-core/src/query/balance.rs | 48 +++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 79c6c6a2c87..8e30e786d51 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -32,7 +32,10 @@ use futures::{ StreamExt, TryStreamExt, }; -use tracing::debug; +use tracing::{ + debug, + error, +}; pub mod asset_query; @@ -57,10 +60,24 @@ impl ReadView { ) .coins() .map(|res| res.map(|coins| coins.amount())) - .try_fold(0u64, |balance, amount| { + .try_fold(0u128, |balance, amount| { async move { // Increase the balance - Ok(balance.saturating_add(amount)) + let maybe_new_balance = balance.checked_add(amount as u128); + match maybe_new_balance { + Some(new_balance) => Ok(new_balance), + None => { + // TODO[RC]: This means that we were not able to update the balances, due to overflow. + // This is a fatal error, because the balances are not consistent with the actual state of the chain. + // However, if we bail here, a lot of integration tests will start failing, because they often + // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. + error!( + %asset_id, + prev_balance=%balance, + "unable to change balance due to overflow"); + Ok(balance.saturating_add(amount as u128)) + } + } } }) .await? as TotalBalanceAmount @@ -111,9 +128,28 @@ impl ReadView { let amount: &mut TotalBalanceAmount = amounts_per_asset .entry(*coin.asset_id(base_asset_id)) .or_default(); - // TODO[RC]: checked_add - *amount = amount.saturating_add(coin.amount() as TotalBalanceAmount); - Ok(amounts_per_asset) + let new_amount = + amount.checked_add(coin.amount() as TotalBalanceAmount); + match new_amount { + Some(new_amount) => { + *amount = new_amount; + Ok(amounts_per_asset) + } + None => { + // TODO[RC]: This means that we were not able to update the balances, due to overflow. + // This is a fatal error, because the balances are not consistent with the actual state of the chain. + // However, if we bail here, a lot of integration tests will start failing, because they often + // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. + error!( + asset_id=%coin.asset_id(base_asset_id), + prev_balance=%amount, + "unable to change balance due to overflow"); + let new_amount = amount + .saturating_add(coin.amount() as TotalBalanceAmount); + *amount = new_amount; + Ok(amounts_per_asset) + } + } }, ) .into_stream() From 4f307e3d4749abcafe04a3b95b7e221f0aa5e6b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 11 Nov 2024 12:47:23 +0100 Subject: [PATCH 086/111] Mention an balance overflow follow-up issue in the comments --- .../src/graphql_api/worker_service.rs | 5 +- crates/fuel-core/src/query/balance.rs | 54 ++++++------------- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 68589dfc8a1..6c9141b7ffe 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -336,10 +336,7 @@ where if let Err(err) = process_balances_update(event.deref(), block_st_transaction, balances_enabled) { - // TODO[RC]: This means that we were not able to update the balances, most likely due to overflow. - // This is a fatal error, because the balances are not consistent with the actual state of the chain. - // However, if we bail here, a lot of integration tests will start failing, because they often - // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. + // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 tracing::error!(%err, "Processing balances") } match event.deref() { diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 8e30e786d51..4c3a5cc9e84 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -60,25 +60,15 @@ impl ReadView { ) .coins() .map(|res| res.map(|coins| coins.amount())) - .try_fold(0u128, |balance, amount| { - async move { - // Increase the balance - let maybe_new_balance = balance.checked_add(amount as u128); - match maybe_new_balance { - Some(new_balance) => Ok(new_balance), - None => { - // TODO[RC]: This means that we were not able to update the balances, due to overflow. - // This is a fatal error, because the balances are not consistent with the actual state of the chain. - // However, if we bail here, a lot of integration tests will start failing, because they often - // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. - error!( - %asset_id, - prev_balance=%balance, - "unable to change balance due to overflow"); - Ok(balance.saturating_add(amount as u128)) - } - } - } + .try_fold(0u128, |balance, amount| async move { + Ok(balance.checked_add(amount as u128).unwrap_or_else(|| { + // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 + error!( + %asset_id, + prev_balance=%balance, + "unable to change balance due to overflow"); + u128::MAX + })) }) .await? as TotalBalanceAmount }; @@ -128,28 +118,18 @@ impl ReadView { let amount: &mut TotalBalanceAmount = amounts_per_asset .entry(*coin.asset_id(base_asset_id)) .or_default(); - let new_amount = - amount.checked_add(coin.amount() as TotalBalanceAmount); - match new_amount { - Some(new_amount) => { - *amount = new_amount; - Ok(amounts_per_asset) - } - None => { - // TODO[RC]: This means that we were not able to update the balances, due to overflow. - // This is a fatal error, because the balances are not consistent with the actual state of the chain. - // However, if we bail here, a lot of integration tests will start failing, because they often - // use transactions that do not necessarily care about asset balances. This needs to be addressed in a separate PR. + let new_amount = amount + .checked_add(coin.amount() as TotalBalanceAmount) + .unwrap_or_else(|| { + // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 error!( asset_id=%coin.asset_id(base_asset_id), prev_balance=%amount, "unable to change balance due to overflow"); - let new_amount = amount - .saturating_add(coin.amount() as TotalBalanceAmount); - *amount = new_amount; - Ok(amounts_per_asset) - } - } + u128::MAX + }); + *amount = new_amount; + Ok(amounts_per_asset) }, ) .into_stream() From 66d594829a03f3f537e7cbd9d7d25ce899541401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Mon, 11 Nov 2024 12:48:59 +0100 Subject: [PATCH 087/111] Remove the `TODO` comment --- crates/fuel-core/src/database/database_description/off_chain.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs index a29becf2d2a..1f339c50f3c 100644 --- a/crates/fuel-core/src/database/database_description/off_chain.rs +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -11,7 +11,6 @@ impl DatabaseDescription for OffChain { type Column = fuel_core_graphql_api::storage::Column; type Height = BlockHeight; - // TODO[RC]: Do we bump this due to extended metadata? fn version() -> u32 { 0 } From 9a07cd2c3b27bf9502c97060f82d03c8fb75e6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 15 Nov 2024 10:57:59 +0100 Subject: [PATCH 088/111] Prevent the metadata from being overwritten with incorrect version --- crates/fuel-core/src/database.rs | 169 +++++++++++++++++++++++++++++-- 1 file changed, 162 insertions(+), 7 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 6d4d9752f7c..34e453d29b1 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -24,6 +24,7 @@ use crate::{ KeyValueView, }, }; +use database_description::IndexationKind; use fuel_core_chain_config::TableEntry; use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; use fuel_core_services::SharedMutex; @@ -58,6 +59,7 @@ use fuel_core_types::{ }; use itertools::Itertools; use std::{ + borrow::Cow, fmt::Debug, sync::Arc, }; @@ -482,15 +484,13 @@ where ConflictPolicy::Overwrite, changes, ); + let maybe_current_metadata = transaction + .storage_as_mut::>() + .get(&())?; + let metadata = update_metadata::(maybe_current_metadata, new_height); transaction .storage_as_mut::>() - .insert( - &(), - &DatabaseMetadata::V1 { - version: Description::version(), - height: new_height, - }, - )?; + .insert(&(), &metadata)?; transaction.into_changes() } else { @@ -508,6 +508,37 @@ where Ok(()) } + +fn update_metadata( + maybe_current_metadata: Option< + Cow::Height>>, + >, + new_height: ::Height, +) -> DatabaseMetadata<::Height> +where + Description: DatabaseDescription, +{ + let updated_metadata = match maybe_current_metadata.as_ref() { + Some(metadata) => match metadata.as_ref() { + DatabaseMetadata::V1 { .. } => DatabaseMetadata::V1 { + version: Description::version(), + height: new_height, + }, + DatabaseMetadata::V2 { .. } => DatabaseMetadata::V2 { + version: Description::version(), + height: new_height, + indexation_availability: IndexationKind::all().collect(), + }, + }, + None => DatabaseMetadata::V2 { + version: Description::version(), + height: new_height, + indexation_availability: IndexationKind::all().collect(), + }, + }; + updated_metadata +} + #[cfg(feature = "rocksdb")] pub fn convert_to_rocksdb_direction(direction: IterDirection) -> rocksdb::Direction { match direction { @@ -524,9 +555,11 @@ mod tests { Database, }; use fuel_core_storage::{ + kv_store::StorageColumn, tables::FuelBlocks, StorageAsMut, }; + use strum::EnumCount; fn column_keys_not_exceed_count() where @@ -1083,4 +1116,126 @@ mod tests { // rocks db fails test(db); } + + #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] + struct HeightMock(u64); + impl DatabaseHeight for HeightMock { + fn as_u64(&self) -> u64 { + 1 + } + + fn advance_height(&self) -> Option { + None + } + + fn rollback_height(&self) -> Option { + None + } + } + + const MOCK_VERSION: u32 = 0; + + #[derive(EnumCount, enum_iterator::Sequence, Debug, Clone, Copy)] + enum ColumnMock { + Column1, + } + + impl StorageColumn for ColumnMock { + fn name(&self) -> String { + "column".to_string() + } + + fn id(&self) -> u32 { + 42 + } + } + + #[derive(Debug, Clone, Copy)] + struct DatabaseDescriptionMock; + impl DatabaseDescription for DatabaseDescriptionMock { + type Column = ColumnMock; + + type Height = HeightMock; + + fn version() -> u32 { + MOCK_VERSION + } + + fn name() -> String { + "mock".to_string() + } + + fn metadata_column() -> Self::Column { + Self::Column::Column1 + } + + fn prefix(_: &Self::Column) -> Option { + None + } + } + + #[test] + fn update_metadata_preserves_v1() { + let current_metadata: DatabaseMetadata = DatabaseMetadata::V1 { + version: MOCK_VERSION, + height: HeightMock(1), + }; + let new_metadata = update_metadata::( + Some(Cow::Borrowed(¤t_metadata)), + HeightMock(2), + ); + + match new_metadata { + DatabaseMetadata::V1 { version, height } => { + assert_eq!(version, current_metadata.version()); + assert_eq!(height, HeightMock(2)); + } + DatabaseMetadata::V2 { .. } => panic!("should be V1"), + } + } + + #[test] + fn update_metadata_preserves_v2() { + let current_metadata: DatabaseMetadata = DatabaseMetadata::V2 { + version: MOCK_VERSION, + height: HeightMock(1), + indexation_availability: IndexationKind::all().collect(), + }; + let new_metadata = update_metadata::( + Some(Cow::Borrowed(¤t_metadata)), + HeightMock(2), + ); + + match new_metadata { + DatabaseMetadata::V1 { .. } => panic!("should be V2"), + DatabaseMetadata::V2 { + version, + height, + indexation_availability, + } => { + assert_eq!(version, current_metadata.version()); + assert_eq!(height, HeightMock(2)); + assert_eq!(indexation_availability, IndexationKind::all().collect()); + } + } + } + + #[test] + fn update_metadata_none_becomes_v2() { + let new_metadata = + update_metadata::(None, HeightMock(2)); + + match new_metadata { + DatabaseMetadata::V1 { .. } => panic!("should be V2"), + DatabaseMetadata::V2 { + version, + height, + indexation_availability, + } => { + assert_eq!(version, MOCK_VERSION); + assert_eq!(height, HeightMock(2)); + assert_eq!(indexation_availability, IndexationKind::all().collect()); + } + } + } } From 35663aeb65903e00c52aaee8d28a21b17b0f1e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 10:28:50 +0100 Subject: [PATCH 089/111] Move metadata tests to a dedicated module --- crates/fuel-core/src/database.rs | 203 +++++++++++++++++-------------- 1 file changed, 109 insertions(+), 94 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 34e453d29b1..deec924e6f3 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -1117,124 +1117,139 @@ mod tests { test(db); } - #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] - struct HeightMock(u64); - impl DatabaseHeight for HeightMock { - fn as_u64(&self) -> u64 { - 1 - } + mod metadata { + use std::borrow::Cow; - fn advance_height(&self) -> Option { - None - } + use fuel_core_storage::kv_store::StorageColumn; + use strum::EnumCount; - fn rollback_height(&self) -> Option { - None - } - } + use super::{ + database_description::DatabaseDescription, + update_metadata, + DatabaseHeight, + DatabaseMetadata, + IndexationKind, + }; - const MOCK_VERSION: u32 = 0; + #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] + struct HeightMock(u64); + impl DatabaseHeight for HeightMock { + fn as_u64(&self) -> u64 { + 1 + } - #[derive(EnumCount, enum_iterator::Sequence, Debug, Clone, Copy)] - enum ColumnMock { - Column1, - } + fn advance_height(&self) -> Option { + None + } - impl StorageColumn for ColumnMock { - fn name(&self) -> String { - "column".to_string() + fn rollback_height(&self) -> Option { + None + } } - fn id(&self) -> u32 { - 42 - } - } + const MOCK_VERSION: u32 = 0; - #[derive(Debug, Clone, Copy)] - struct DatabaseDescriptionMock; - impl DatabaseDescription for DatabaseDescriptionMock { - type Column = ColumnMock; + #[derive(EnumCount, enum_iterator::Sequence, Debug, Clone, Copy)] + enum ColumnMock { + Column1, + } - type Height = HeightMock; + impl StorageColumn for ColumnMock { + fn name(&self) -> String { + "column".to_string() + } - fn version() -> u32 { - MOCK_VERSION + fn id(&self) -> u32 { + 42 + } } - fn name() -> String { - "mock".to_string() - } + #[derive(Debug, Clone, Copy)] + struct DatabaseDescriptionMock; + impl DatabaseDescription for DatabaseDescriptionMock { + type Column = ColumnMock; - fn metadata_column() -> Self::Column { - Self::Column::Column1 - } + type Height = HeightMock; + + fn version() -> u32 { + MOCK_VERSION + } - fn prefix(_: &Self::Column) -> Option { - None + fn name() -> String { + "mock".to_string() + } + + fn metadata_column() -> Self::Column { + Self::Column::Column1 + } + + fn prefix(_: &Self::Column) -> Option { + None + } } - } - #[test] - fn update_metadata_preserves_v1() { - let current_metadata: DatabaseMetadata = DatabaseMetadata::V1 { - version: MOCK_VERSION, - height: HeightMock(1), - }; - let new_metadata = update_metadata::( - Some(Cow::Borrowed(¤t_metadata)), - HeightMock(2), - ); + #[test] + fn update_metadata_preserves_v1() { + let current_metadata: DatabaseMetadata = DatabaseMetadata::V1 { + version: MOCK_VERSION, + height: HeightMock(1), + }; + let new_metadata = update_metadata::( + Some(Cow::Borrowed(¤t_metadata)), + HeightMock(2), + ); - match new_metadata { - DatabaseMetadata::V1 { version, height } => { - assert_eq!(version, current_metadata.version()); - assert_eq!(height, HeightMock(2)); + match new_metadata { + DatabaseMetadata::V1 { version, height } => { + assert_eq!(version, current_metadata.version()); + assert_eq!(height, HeightMock(2)); + } + DatabaseMetadata::V2 { .. } => panic!("should be V1"), } - DatabaseMetadata::V2 { .. } => panic!("should be V1"), } - } - #[test] - fn update_metadata_preserves_v2() { - let current_metadata: DatabaseMetadata = DatabaseMetadata::V2 { - version: MOCK_VERSION, - height: HeightMock(1), - indexation_availability: IndexationKind::all().collect(), - }; - let new_metadata = update_metadata::( - Some(Cow::Borrowed(¤t_metadata)), - HeightMock(2), - ); + #[test] + fn update_metadata_preserves_v2() { + let current_metadata: DatabaseMetadata = DatabaseMetadata::V2 { + version: MOCK_VERSION, + height: HeightMock(1), + indexation_availability: IndexationKind::all().collect(), + }; + let new_metadata = update_metadata::( + Some(Cow::Borrowed(¤t_metadata)), + HeightMock(2), + ); - match new_metadata { - DatabaseMetadata::V1 { .. } => panic!("should be V2"), - DatabaseMetadata::V2 { - version, - height, - indexation_availability, - } => { - assert_eq!(version, current_metadata.version()); - assert_eq!(height, HeightMock(2)); - assert_eq!(indexation_availability, IndexationKind::all().collect()); + match new_metadata { + DatabaseMetadata::V1 { .. } => panic!("should be V2"), + DatabaseMetadata::V2 { + version, + height, + indexation_availability, + } => { + assert_eq!(version, current_metadata.version()); + assert_eq!(height, HeightMock(2)); + assert_eq!(indexation_availability, IndexationKind::all().collect()); + } } } - } - #[test] - fn update_metadata_none_becomes_v2() { - let new_metadata = - update_metadata::(None, HeightMock(2)); - - match new_metadata { - DatabaseMetadata::V1 { .. } => panic!("should be V2"), - DatabaseMetadata::V2 { - version, - height, - indexation_availability, - } => { - assert_eq!(version, MOCK_VERSION); - assert_eq!(height, HeightMock(2)); - assert_eq!(indexation_availability, IndexationKind::all().collect()); + #[test] + fn update_metadata_none_becomes_v2() { + let new_metadata = + update_metadata::(None, HeightMock(2)); + + match new_metadata { + DatabaseMetadata::V1 { .. } => panic!("should be V2"), + DatabaseMetadata::V2 { + version, + height, + indexation_availability, + } => { + assert_eq!(version, MOCK_VERSION); + assert_eq!(height, HeightMock(2)); + assert_eq!(indexation_availability, IndexationKind::all().collect()); + } } } } From 179cf6984a40cbf707814401211b9b628d5c2330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 10:49:24 +0100 Subject: [PATCH 090/111] Keep the previous indexation availability when updating metadata --- crates/fuel-core/src/database.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index deec924e6f3..8db380f51b9 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -524,10 +524,13 @@ where version: Description::version(), height: new_height, }, - DatabaseMetadata::V2 { .. } => DatabaseMetadata::V2 { + DatabaseMetadata::V2 { + indexation_availability, + .. + } => DatabaseMetadata::V2 { version: Description::version(), height: new_height, - indexation_availability: IndexationKind::all().collect(), + indexation_availability: indexation_availability.clone(), }, }, None => DatabaseMetadata::V2 { @@ -554,12 +557,6 @@ mod tests { database_description::DatabaseDescription, Database, }; - use fuel_core_storage::{ - kv_store::StorageColumn, - tables::FuelBlocks, - StorageAsMut, - }; - use strum::EnumCount; fn column_keys_not_exceed_count() where @@ -1118,7 +1115,10 @@ mod tests { } mod metadata { - use std::borrow::Cow; + use std::{ + borrow::Cow, + collections::HashSet, + }; use fuel_core_storage::kv_store::StorageColumn; use strum::EnumCount; @@ -1210,10 +1210,12 @@ mod tests { #[test] fn update_metadata_preserves_v2() { + let available_indexation = HashSet::new(); + let current_metadata: DatabaseMetadata = DatabaseMetadata::V2 { version: MOCK_VERSION, height: HeightMock(1), - indexation_availability: IndexationKind::all().collect(), + indexation_availability: available_indexation.clone(), }; let new_metadata = update_metadata::( Some(Cow::Borrowed(¤t_metadata)), @@ -1229,7 +1231,7 @@ mod tests { } => { assert_eq!(version, current_metadata.version()); assert_eq!(height, HeightMock(2)); - assert_eq!(indexation_availability, IndexationKind::all().collect()); + assert_eq!(indexation_availability, available_indexation); } } } From 1d4005673386e8dbb92ec48138c2dc338bf1f291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 13:39:02 +0100 Subject: [PATCH 091/111] Add tests for balance with (non)retryable messages --- tests/tests/balances.rs | 95 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index a5892b434eb..20eb662914c 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -132,6 +132,101 @@ async fn balance() { assert_eq!(balance, 449); } +#[tokio::test] +async fn balance_messages_only() { + let owner = Address::default(); + let asset_id = AssetId::BASE; + + const RETRYABLE: &[u8] = &[1]; + const NON_RETRYABLE: &[u8] = &[]; + + // setup config + let state_config = StateConfig { + contracts: vec![], + coins: vec![], + messages: vec![ + (owner, 60, NON_RETRYABLE), + (owner, 200, RETRYABLE), + (owner, 90, NON_RETRYABLE), + ] + .into_iter() + .enumerate() + .map(|(nonce, (owner, amount, data))| MessageConfig { + sender: owner, + recipient: owner, + nonce: (nonce as u64).into(), + amount, + data: data.to_vec(), + da_height: DaBlockHeight::from(0usize), + }) + .collect(), + ..Default::default() + }; + let config = Config::local_node_with_state_config(state_config); + + // setup server & client + let srv = FuelService::new_node(config).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + + // run test + const NON_RETRYABLE_AMOUNT: u128 = 60 + 90; + let balance = client.balance(&owner, Some(&asset_id)).await.unwrap(); + assert_eq!(balance, NON_RETRYABLE_AMOUNT); +} + +#[tokio::test] +async fn balances_messages_only() { + let owner = Address::default(); + + const RETRYABLE: &[u8] = &[1]; + const NON_RETRYABLE: &[u8] = &[]; + + // setup config + let state_config = StateConfig { + contracts: vec![], + coins: vec![], + messages: vec![ + (owner, 60, NON_RETRYABLE), + (owner, 200, RETRYABLE), + (owner, 90, NON_RETRYABLE), + ] + .into_iter() + .enumerate() + .map(|(nonce, (owner, amount, data))| MessageConfig { + sender: owner, + recipient: owner, + nonce: (nonce as u64).into(), + amount, + data: data.to_vec(), + da_height: DaBlockHeight::from(0usize), + }) + .collect(), + ..Default::default() + }; + let config = Config::local_node_with_state_config(state_config); + + // setup server & client + let srv = FuelService::new_node(config).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + + // run test + const NON_RETRYABLE_AMOUNT: u128 = 60 + 90; + let balances = client + .balances( + &owner, + PaginationRequest { + cursor: None, + results: 10, + direction: PageDirection::Forward, + }, + ) + .await + .unwrap(); + assert_eq!(balances.results.len(), 1); + let messages_balance = balances.results[0].amount; + assert_eq!(messages_balance, NON_RETRYABLE_AMOUNT); +} + #[tokio::test] async fn first_5_balances() { let owner = Address::from([10u8; 32]); From bf8334637c2dc95f2f019e4c814f1d0fd8f37026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 13:39:50 +0100 Subject: [PATCH 092/111] Refactor the `balances()` off_chain function --- .../service/adapters/graphql_api/off_chain.rs | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 67bb2741369..d580ad558a2 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -244,26 +244,22 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let key = balance_key?; let asset_id = key.asset_id(); - let messages = if base_asset_id == asset_id { - self.storage_as_ref::() - .get(owner)? - .unwrap_or_default() - .into_owned() as TotalBalanceAmount - } else { - 0 - }; - let coins = self .storage_as_ref::() .get(&key)? .unwrap_or_default() .into_owned() as TotalBalanceAmount; - let total = coins.checked_add(messages).ok_or(anyhow::anyhow!( - "Total balance overflow: coins: {coins}, messages: {messages}" - ))?; - debug!(%owner, %asset_id, %total, "balance entry"); - balances.insert(*asset_id, total); + balances.insert(asset_id.clone(), coins); + } + + if let Some(messages) = self.storage_as_ref::().get(owner)? { + balances + .entry(*base_asset_id) + .and_modify(|current| { + *current += *messages; + }) + .or_insert(*messages); } Ok(balances) From 4805aa2b179a5eebc5a5a3caf201bbec26898400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 17:10:14 +0100 Subject: [PATCH 093/111] Make balance updates less generic but more readable --- .../src/graphql_api/storage/balances.rs | 22 +- .../src/graphql_api/worker_service.rs | 211 ++++++++++-------- .../src/query/balance/asset_query.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 26 ++- crates/types/src/entities/relayer/message.rs | 5 + 5 files changed, 161 insertions(+), 105 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 10aec2862e7..a0576bc22c7 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -23,16 +23,16 @@ use rand::{ pub type ItemAmount = u64; pub type TotalBalanceAmount = u128; -double_key!(BalancesKey, Address, address, AssetId, asset_id); -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> BalancesKey { - let mut bytes = [0u8; BalancesKey::LEN]; +double_key!(CoinBalancesKey, Address, address, AssetId, asset_id); +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> CoinBalancesKey { + let mut bytes = [0u8; CoinBalancesKey::LEN]; rng.fill_bytes(bytes.as_mut()); - BalancesKey::from_array(bytes) + CoinBalancesKey::from_array(bytes) } } -impl core::fmt::Display for BalancesKey { +impl core::fmt::Display for CoinBalancesKey { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { write!(f, "address={} asset_id={}", self.address(), self.asset_id()) } @@ -42,7 +42,7 @@ impl core::fmt::Display for BalancesKey { pub struct CoinBalances; impl Mappable for CoinBalances { - type Key = BalancesKey; + type Key = CoinBalancesKey; type OwnedKey = Self::Key; type Value = TotalBalanceAmount; type OwnedValue = Self::Value; @@ -57,13 +57,19 @@ impl TableWithBlueprint for CoinBalances { } } +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct MessageBalance { + pub retryable: TotalBalanceAmount, + pub non_retryable: TotalBalanceAmount, +} + /// This table stores the balances of messages per owner. pub struct MessageBalances; impl Mappable for MessageBalances { type Key = Address; type OwnedKey = Self::Key; - type Value = TotalBalanceAmount; + type Value = MessageBalance; type OwnedValue = Self::Value; } diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 6c9141b7ffe..7550a36b644 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -2,8 +2,9 @@ use super::{ da_compression::da_compress_block, storage::{ balances::{ - BalancesKey, + CoinBalancesKey, ItemAmount, + MessageBalance, TotalBalanceAmount, }, old::{ @@ -206,88 +207,127 @@ where } } -trait DatabaseItemWithAmount { - type Storage: Mappable; - - fn key(&self) -> ::Key; - fn amount(&self) -> ItemAmount; -} - -impl DatabaseItemWithAmount for &Coin { - type Storage = CoinBalances; - - fn key(&self) -> ::Key { - BalancesKey::new(&self.owner, &self.asset_id) - } - - fn amount(&self) -> ItemAmount { - self.amount +// TODO[RC]: A lot of duplication below, consider refactoring. +fn increase_message_balance( + block_st_transaction: &mut T, + message: &Message, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + let key = message.recipient(); + let storage = block_st_transaction.storage::(); + let MessageBalance { + mut retryable, + mut non_retryable, + } = *storage.get(&key)?.unwrap_or_default(); + + if message.has_retryable_amount() { + retryable += message.amount() as u128; + } else { + non_retryable += message.amount() as u128; } -} -impl DatabaseItemWithAmount for &Message { - type Storage = MessageBalances; + let new_balance = MessageBalance { + retryable, + non_retryable, + }; - fn key(&self) -> ::Key { - *self.recipient() - } - - fn amount(&self) -> ItemAmount { - (**self).amount() - } + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &new_balance)?) } -trait BalanceIndexationUpdater: DatabaseItemWithAmount { - type TotalBalance: From<::OwnedValue> + core::fmt::Display; - - fn update_balances( - &self, - tx: &mut T, - updater: UpdaterFn, - ) -> StorageResult<()> - where - ::Key: Sized + core::fmt::Display, - ::OwnedValue: Default + core::fmt::Display, - UpdaterFn: Fn(Self::TotalBalance, ItemAmount) -> Option, - T: OffChainDatabaseTransaction + StorageMutate, - ::Value: - From<::TotalBalance>, - fuel_core_storage::Error: From<>::Error>, - { - let key = self.key(); - let amount = self.amount(); - let storage = tx.storage::(); - let current_balance = storage.get(&key)?.unwrap_or_default(); - let prev_balance = current_balance.clone(); - match updater(current_balance.as_ref().clone().into(), amount) { - Some(new_balance) => { - debug!( - %key, - %amount, - %prev_balance, - %new_balance, - "changing balance"); - - let storage = tx.storage::(); - Ok(storage.insert(&key, &new_balance.into())?) +fn decrease_message_balance( + block_st_transaction: &mut T, + message: &Message, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + let key = message.recipient(); + let storage = block_st_transaction.storage::(); + let MessageBalance { + mut retryable, + mut non_retryable, + } = *storage.get(&key)?.unwrap_or_default(); + + if message.has_retryable_amount() { + let maybe_new_amount = retryable.checked_sub(message.amount() as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + let new_balance = MessageBalance { + retryable: new_amount, + non_retryable, + }; + return Ok(storage.insert(&key, &new_balance)?); } None => { - error!( - %key, - %amount, - %prev_balance, - "unable to change balance due to overflow"); - Err(anyhow::anyhow!("unable to change balance due to overflow").into()) + error!(%retryable, amount=%message.amount(), "Retryable balance would go below 0"); + return Ok(()) + } + } + } else { + let maybe_new_amount = non_retryable.checked_sub(message.amount() as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + let new_balance = MessageBalance { + retryable: new_amount, + non_retryable, + }; + return Ok(storage.insert(&key, &new_balance)?); + } + None => { + error!(%retryable, amount=%message.amount(), "Non-retryable balance would go below 0"); + return Ok(()) } } } } -impl BalanceIndexationUpdater for &Coin { - type TotalBalance = TotalBalanceAmount; +fn increase_coin_balance( + block_st_transaction: &mut T, + coin: &Coin, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); + let storage = block_st_transaction.storage::(); + let mut amount = *storage.get(&key)?.unwrap_or_default(); + amount += coin.amount as u128; + + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &amount)?) } -impl BalanceIndexationUpdater for &Message { - type TotalBalance = TotalBalanceAmount; + +fn decrease_coin_balance( + block_st_transaction: &mut T, + coin: &Coin, +) -> StorageResult<()> +where + T: OffChainDatabaseTransaction, +{ + let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); + let storage = block_st_transaction.storage::(); + let mut current_amount = *storage.get(&key)?.unwrap_or_default(); + + let maybe_new_amount = current_amount.checked_sub(coin.amount as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &new_amount)?) + } + None => { + error!( + owner=%coin.owner, + asset_id=%coin.asset_id, + %current_amount, + coin_amount=%coin.amount, "Coin balance would go below 0"); + Ok(()) + } + } } fn process_balances_update( @@ -301,24 +341,21 @@ where if !balances_enabled { return Ok(()); } + match event { - Event::MessageImported(message) => message - .update_balances(block_st_transaction, |balance, amount| { - balance.checked_add(amount as TotalBalanceAmount) - }), - Event::MessageConsumed(message) => message - .update_balances(block_st_transaction, |balance, amount| { - balance.checked_sub(amount as TotalBalanceAmount) - }), - Event::CoinCreated(coin) => coin - .update_balances(block_st_transaction, |balance, amount| { - balance.checked_add(amount as TotalBalanceAmount) - }), - Event::CoinConsumed(coin) => coin - .update_balances(block_st_transaction, |balance, amount| { - balance.checked_sub(amount as TotalBalanceAmount) - }), - Event::ForcedTransactionFailed { .. } => Ok(()), + Event::MessageImported(message) => { + increase_message_balance(block_st_transaction, message) + } + Event::MessageConsumed(message) => { + decrease_message_balance(block_st_transaction, message) + } + Event::CoinCreated(coin) => increase_coin_balance(block_st_transaction, coin), + Event::CoinConsumed(coin) => decrease_coin_balance(block_st_transaction, coin), + Event::ForcedTransactionFailed { + id, + block_height, + failure, + } => Ok(()), } } diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index 13a289ec1e4..a7ddd4a5f05 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -175,7 +175,7 @@ impl<'a> AssetsQuery<'a> { .try_flatten() .filter(|result| { if let Ok(message) = result { - message.data().is_empty() + !message.has_retryable_amount() } else { true } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index d580ad558a2..7dd5cb7794c 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -23,8 +23,9 @@ use crate::{ }, graphql_api::storage::{ balances::{ - BalancesKey, + CoinBalancesKey, CoinBalances, + MessageBalance, MessageBalances, TotalBalanceAmount, }, @@ -211,22 +212,25 @@ impl OffChainDatabase for OffChainIterableKeyValueView { ) -> StorageResult { let coins = self .storage_as_ref::() - .get(&BalancesKey::new(owner, asset_id))? + .get(&CoinBalancesKey::new(owner, asset_id))? .unwrap_or_default() .into_owned() as TotalBalanceAmount; if base_asset_id == asset_id { - let messages = self + let MessageBalance { + retryable: _, // TODO[RC]: Handle this + non_retryable, + } = self .storage_as_ref::() .get(owner)? .unwrap_or_default() - .into_owned() as TotalBalanceAmount; + .into_owned(); - let total = coins.checked_add(messages).ok_or(anyhow::anyhow!( - "Total balance overflow: coins: {coins}, messages: {messages}" + let total = coins.checked_add(non_retryable).ok_or(anyhow::anyhow!( + "Total balance overflow: coins: {coins}, messages: {non_retryable}" ))?; - debug!(%coins, %messages, total, "total balance"); + debug!(%coins, %non_retryable, total, "total balance"); Ok(total) } else { debug!(%coins, "total balance"); @@ -254,12 +258,16 @@ impl OffChainDatabase for OffChainIterableKeyValueView { } if let Some(messages) = self.storage_as_ref::().get(owner)? { + let MessageBalance { + retryable: _, + non_retryable, + } = *messages; balances .entry(*base_asset_id) .and_modify(|current| { - *current += *messages; + *current += non_retryable; }) - .or_insert(*messages); + .or_insert(non_retryable); } Ok(balances) diff --git a/crates/types/src/entities/relayer/message.rs b/crates/types/src/entities/relayer/message.rs index b058b6ff75c..9ed46ffbbfc 100644 --- a/crates/types/src/entities/relayer/message.rs +++ b/crates/types/src/entities/relayer/message.rs @@ -135,6 +135,11 @@ impl Message { } } + /// Returns true if the message has retryable amount. + pub fn has_retryable_amount(&self) -> bool { + !self.data().is_empty() + } + /// Set the message data #[cfg(any(test, feature = "test-helpers"))] pub fn set_data(&mut self, data: Vec) { From be9fdda5c2a3458a0208e8b793840b74c058b76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 17:37:46 +0100 Subject: [PATCH 094/111] Introduce `IndexationError` --- .../src/graphql_api/worker_service.rs | 94 ++++++++++++++----- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 7550a36b644..6f51ad3fd36 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -81,6 +81,8 @@ use fuel_core_types::{ CoinPredicate, CoinSigned, }, + Address, + AssetId, Contract, Input, Output, @@ -207,11 +209,43 @@ where } } +#[derive(derive_more::From, derive_more::Display)] +enum IndexationError { + #[display( + fmt = "Coin balance would underflow for owner: {}, asset_id: {}, current_amount: {}, requested_deduction: {}", + owner, + asset_id, + current_amount, + requested_deduction + )] + CoinBalanceWouldUnderflow { + owner: Address, + asset_id: AssetId, + current_amount: u128, + requested_deduction: u128, + }, + #[display( + fmt = "Message balance would underflow for owner: {}, current_amount: {}, requested_deduction: {}, retryable: {}", + owner, + current_amount, + requested_deduction, + retryable + )] + MessageBalanceWouldUnderflow { + owner: Address, + current_amount: u128, + requested_deduction: u128, + retryable: bool, + }, + #[from] + StorageError(StorageError), +} + // TODO[RC]: A lot of duplication below, consider refactoring. fn increase_message_balance( block_st_transaction: &mut T, message: &Message, -) -> StorageResult<()> +) -> Result<(), IndexationError> where T: OffChainDatabaseTransaction, { @@ -240,7 +274,7 @@ where fn decrease_message_balance( block_st_transaction: &mut T, message: &Message, -) -> StorageResult<()> +) -> Result<(), IndexationError> where T: OffChainDatabaseTransaction, { @@ -263,8 +297,12 @@ where return Ok(storage.insert(&key, &new_balance)?); } None => { - error!(%retryable, amount=%message.amount(), "Retryable balance would go below 0"); - return Ok(()) + return Err(IndexationError::MessageBalanceWouldUnderflow { + owner: message.recipient().clone(), + current_amount: retryable, + requested_deduction: message.amount() as u128, + retryable: true, + }); } } } else { @@ -279,8 +317,12 @@ where return Ok(storage.insert(&key, &new_balance)?); } None => { - error!(%retryable, amount=%message.amount(), "Non-retryable balance would go below 0"); - return Ok(()) + return Err(IndexationError::MessageBalanceWouldUnderflow { + owner: message.recipient().clone(), + current_amount: retryable, + requested_deduction: message.amount() as u128, + retryable: false, + }); } } } @@ -289,7 +331,7 @@ where fn increase_coin_balance( block_st_transaction: &mut T, coin: &Coin, -) -> StorageResult<()> +) -> Result<(), IndexationError> where T: OffChainDatabaseTransaction, { @@ -305,7 +347,7 @@ where fn decrease_coin_balance( block_st_transaction: &mut T, coin: &Coin, -) -> StorageResult<()> +) -> Result<(), IndexationError> where T: OffChainDatabaseTransaction, { @@ -319,14 +361,12 @@ where let storage = block_st_transaction.storage::(); Ok(storage.insert(&key, &new_amount)?) } - None => { - error!( - owner=%coin.owner, - asset_id=%coin.asset_id, - %current_amount, - coin_amount=%coin.amount, "Coin balance would go below 0"); - Ok(()) - } + None => Err(IndexationError::CoinBalanceWouldUnderflow { + owner: coin.owner.clone(), + asset_id: coin.asset_id.clone(), + current_amount, + requested_deduction: coin.amount as u128, + }), } } @@ -334,7 +374,7 @@ fn process_balances_update( event: &Event, block_st_transaction: &mut T, balances_enabled: bool, -) -> StorageResult<()> +) -> Result<(), IndexationError> where T: OffChainDatabaseTransaction, { @@ -370,12 +410,22 @@ where T: OffChainDatabaseTransaction, { for event in events { - if let Err(err) = - process_balances_update(event.deref(), block_st_transaction, balances_enabled) - { - // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 - tracing::error!(%err, "Processing balances") + match process_balances_update( + event.deref(), + block_st_transaction, + balances_enabled, + ) { + Ok(()) => (), + Err(IndexationError::StorageError(err)) => { + return Err(err.into()); + } + Err(err @ IndexationError::CoinBalanceWouldUnderflow { .. }) + | Err(err @ IndexationError::MessageBalanceWouldUnderflow { .. }) => { + // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 + error!("Balances underflow detected: {}", err); + } } + match event.deref() { Event::MessageImported(message) => { block_st_transaction From 623bb9fb177a8ec039f0b866d50ce83d59be3b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 17:49:22 +0100 Subject: [PATCH 095/111] Extract indexation to a separate module --- crates/fuel-core/src/graphql_api.rs | 1 + .../fuel-core/src/graphql_api/indexation.rs | 219 ++++++++++++++++++ .../src/graphql_api/worker_service.rs | 195 +--------------- .../service/adapters/graphql_api/off_chain.rs | 4 +- 4 files changed, 226 insertions(+), 193 deletions(-) create mode 100644 crates/fuel-core/src/graphql_api/indexation.rs diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 0dc3fe6b8db..97651089bfe 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -11,6 +11,7 @@ use std::{ pub mod api_service; mod da_compression; pub mod database; +pub(crate) mod indexation; pub(crate) mod metrics_extension; pub mod ports; pub mod storage; diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs new file mode 100644 index 00000000000..28974db5935 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -0,0 +1,219 @@ +use fuel_core_storage::{ + Error as StorageError, + Mappable, + Result as StorageResult, + StorageAsMut, + StorageInspect, + StorageMutate, +}; +use fuel_core_types::{ + entities::{ + coins::coin::Coin, + Message, + }, + fuel_tx::{ + Address, + AssetId, + }, + services::executor::Event, +}; + +use super::{ + ports::worker::OffChainDatabaseTransaction, + storage::balances::{ + CoinBalances, + CoinBalancesKey, + MessageBalance, + MessageBalances, + }, +}; + +#[derive(derive_more::From, derive_more::Display)] +pub enum IndexationError { + #[display( + fmt = "Coin balance would underflow for owner: {}, asset_id: {}, current_amount: {}, requested_deduction: {}", + owner, + asset_id, + current_amount, + requested_deduction + )] + CoinBalanceWouldUnderflow { + owner: Address, + asset_id: AssetId, + current_amount: u128, + requested_deduction: u128, + }, + #[display( + fmt = "Message balance would underflow for owner: {}, current_amount: {}, requested_deduction: {}, retryable: {}", + owner, + current_amount, + requested_deduction, + retryable + )] + MessageBalanceWouldUnderflow { + owner: Address, + current_amount: u128, + requested_deduction: u128, + retryable: bool, + }, + #[from] + StorageError(StorageError), +} + +// TODO[RC]: A lot of duplication below, consider refactoring. +fn increase_message_balance( + block_st_transaction: &mut T, + message: &Message, +) -> Result<(), IndexationError> +where + T: OffChainDatabaseTransaction, +{ + let key = message.recipient(); + let storage = block_st_transaction.storage::(); + let MessageBalance { + mut retryable, + mut non_retryable, + } = *storage.get(&key)?.unwrap_or_default(); + + if message.has_retryable_amount() { + retryable.saturating_add(message.amount() as u128); + } else { + non_retryable.saturating_add(message.amount() as u128); + } + + let new_balance = MessageBalance { + retryable, + non_retryable, + }; + + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &new_balance)?) +} + +fn decrease_message_balance( + block_st_transaction: &mut T, + message: &Message, +) -> Result<(), IndexationError> +where + T: OffChainDatabaseTransaction, +{ + let key = message.recipient(); + let storage = block_st_transaction.storage::(); + let MessageBalance { + mut retryable, + mut non_retryable, + } = *storage.get(&key)?.unwrap_or_default(); + + if message.has_retryable_amount() { + let maybe_new_amount = retryable.checked_sub(message.amount() as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + let new_balance = MessageBalance { + retryable: new_amount, + non_retryable, + }; + return Ok(storage.insert(&key, &new_balance)?); + } + None => { + return Err(IndexationError::MessageBalanceWouldUnderflow { + owner: message.recipient().clone(), + current_amount: retryable, + requested_deduction: message.amount() as u128, + retryable: true, + }); + } + } + } else { + let maybe_new_amount = non_retryable.checked_sub(message.amount() as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + let new_balance = MessageBalance { + retryable: new_amount, + non_retryable, + }; + return Ok(storage.insert(&key, &new_balance)?); + } + None => { + return Err(IndexationError::MessageBalanceWouldUnderflow { + owner: message.recipient().clone(), + current_amount: retryable, + requested_deduction: message.amount() as u128, + retryable: false, + }); + } + } + } +} + +fn increase_coin_balance( + block_st_transaction: &mut T, + coin: &Coin, +) -> Result<(), IndexationError> +where + T: OffChainDatabaseTransaction, +{ + let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); + let storage = block_st_transaction.storage::(); + let mut amount = *storage.get(&key)?.unwrap_or_default(); + amount.saturating_add(coin.amount as u128); + + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &amount)?) +} + +fn decrease_coin_balance( + block_st_transaction: &mut T, + coin: &Coin, +) -> Result<(), IndexationError> +where + T: OffChainDatabaseTransaction, +{ + let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); + let storage = block_st_transaction.storage::(); + let mut current_amount = *storage.get(&key)?.unwrap_or_default(); + + let maybe_new_amount = current_amount.checked_sub(coin.amount as u128); + match maybe_new_amount { + Some(new_amount) => { + let storage = block_st_transaction.storage::(); + Ok(storage.insert(&key, &new_amount)?) + } + None => Err(IndexationError::CoinBalanceWouldUnderflow { + owner: coin.owner.clone(), + asset_id: coin.asset_id.clone(), + current_amount, + requested_deduction: coin.amount as u128, + }), + } +} + +pub(crate) fn process_balances_update( + event: &Event, + block_st_transaction: &mut T, + balances_enabled: bool, +) -> Result<(), IndexationError> +where + T: OffChainDatabaseTransaction, +{ + if !balances_enabled { + return Ok(()); + } + + match event { + Event::MessageImported(message) => { + increase_message_balance(block_st_transaction, message) + } + Event::MessageConsumed(message) => { + decrease_message_balance(block_st_transaction, message) + } + Event::CoinCreated(coin) => increase_coin_balance(block_st_transaction, coin), + Event::CoinConsumed(coin) => decrease_coin_balance(block_st_transaction, coin), + Event::ForcedTransactionFailed { + id, + block_height, + failure, + } => Ok(()), + } +} diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 6f51ad3fd36..df1c4618897 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,5 +1,8 @@ +use self::indexation::IndexationError; + use super::{ da_compression::da_compress_block, + indexation, storage::{ balances::{ CoinBalancesKey, @@ -209,196 +212,6 @@ where } } -#[derive(derive_more::From, derive_more::Display)] -enum IndexationError { - #[display( - fmt = "Coin balance would underflow for owner: {}, asset_id: {}, current_amount: {}, requested_deduction: {}", - owner, - asset_id, - current_amount, - requested_deduction - )] - CoinBalanceWouldUnderflow { - owner: Address, - asset_id: AssetId, - current_amount: u128, - requested_deduction: u128, - }, - #[display( - fmt = "Message balance would underflow for owner: {}, current_amount: {}, requested_deduction: {}, retryable: {}", - owner, - current_amount, - requested_deduction, - retryable - )] - MessageBalanceWouldUnderflow { - owner: Address, - current_amount: u128, - requested_deduction: u128, - retryable: bool, - }, - #[from] - StorageError(StorageError), -} - -// TODO[RC]: A lot of duplication below, consider refactoring. -fn increase_message_balance( - block_st_transaction: &mut T, - message: &Message, -) -> Result<(), IndexationError> -where - T: OffChainDatabaseTransaction, -{ - let key = message.recipient(); - let storage = block_st_transaction.storage::(); - let MessageBalance { - mut retryable, - mut non_retryable, - } = *storage.get(&key)?.unwrap_or_default(); - - if message.has_retryable_amount() { - retryable += message.amount() as u128; - } else { - non_retryable += message.amount() as u128; - } - - let new_balance = MessageBalance { - retryable, - non_retryable, - }; - - let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &new_balance)?) -} - -fn decrease_message_balance( - block_st_transaction: &mut T, - message: &Message, -) -> Result<(), IndexationError> -where - T: OffChainDatabaseTransaction, -{ - let key = message.recipient(); - let storage = block_st_transaction.storage::(); - let MessageBalance { - mut retryable, - mut non_retryable, - } = *storage.get(&key)?.unwrap_or_default(); - - if message.has_retryable_amount() { - let maybe_new_amount = retryable.checked_sub(message.amount() as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - let new_balance = MessageBalance { - retryable: new_amount, - non_retryable, - }; - return Ok(storage.insert(&key, &new_balance)?); - } - None => { - return Err(IndexationError::MessageBalanceWouldUnderflow { - owner: message.recipient().clone(), - current_amount: retryable, - requested_deduction: message.amount() as u128, - retryable: true, - }); - } - } - } else { - let maybe_new_amount = non_retryable.checked_sub(message.amount() as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - let new_balance = MessageBalance { - retryable: new_amount, - non_retryable, - }; - return Ok(storage.insert(&key, &new_balance)?); - } - None => { - return Err(IndexationError::MessageBalanceWouldUnderflow { - owner: message.recipient().clone(), - current_amount: retryable, - requested_deduction: message.amount() as u128, - retryable: false, - }); - } - } - } -} - -fn increase_coin_balance( - block_st_transaction: &mut T, - coin: &Coin, -) -> Result<(), IndexationError> -where - T: OffChainDatabaseTransaction, -{ - let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); - let storage = block_st_transaction.storage::(); - let mut amount = *storage.get(&key)?.unwrap_or_default(); - amount += coin.amount as u128; - - let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &amount)?) -} - -fn decrease_coin_balance( - block_st_transaction: &mut T, - coin: &Coin, -) -> Result<(), IndexationError> -where - T: OffChainDatabaseTransaction, -{ - let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); - let storage = block_st_transaction.storage::(); - let mut current_amount = *storage.get(&key)?.unwrap_or_default(); - - let maybe_new_amount = current_amount.checked_sub(coin.amount as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &new_amount)?) - } - None => Err(IndexationError::CoinBalanceWouldUnderflow { - owner: coin.owner.clone(), - asset_id: coin.asset_id.clone(), - current_amount, - requested_deduction: coin.amount as u128, - }), - } -} - -fn process_balances_update( - event: &Event, - block_st_transaction: &mut T, - balances_enabled: bool, -) -> Result<(), IndexationError> -where - T: OffChainDatabaseTransaction, -{ - if !balances_enabled { - return Ok(()); - } - - match event { - Event::MessageImported(message) => { - increase_message_balance(block_st_transaction, message) - } - Event::MessageConsumed(message) => { - decrease_message_balance(block_st_transaction, message) - } - Event::CoinCreated(coin) => increase_coin_balance(block_st_transaction, coin), - Event::CoinConsumed(coin) => decrease_coin_balance(block_st_transaction, coin), - Event::ForcedTransactionFailed { - id, - block_height, - failure, - } => Ok(()), - } -} - /// Process the executor events and update the indexes for the messages and coins. pub fn process_executor_events<'a, Iter, T>( events: Iter, @@ -410,7 +223,7 @@ where T: OffChainDatabaseTransaction, { for event in events { - match process_balances_update( + match indexation::process_balances_update( event.deref(), block_st_transaction, balances_enabled, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 7dd5cb7794c..8a3abdb7059 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -23,8 +23,8 @@ use crate::{ }, graphql_api::storage::{ balances::{ - CoinBalancesKey, CoinBalances, + CoinBalancesKey, MessageBalance, MessageBalances, TotalBalanceAmount, @@ -265,7 +265,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { balances .entry(*base_asset_id) .and_modify(|current| { - *current += non_retryable; + current.saturating_add(non_retryable); }) .or_insert(non_retryable); } From 18f37cc43f60a7f3da3e8be8aaffc96db976323a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Tue, 19 Nov 2024 22:12:02 +0100 Subject: [PATCH 096/111] Fix issue in `decrease_message_balance()` --- .../fuel-core/src/graphql_api/indexation.rs | 21 +++++++++---------- .../src/graphql_api/storage/balances.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index 28974db5935..a3df0c28889 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -70,17 +70,16 @@ where { let key = message.recipient(); let storage = block_st_transaction.storage::(); + let current_balance = storage.get(&key)?.unwrap_or_default(); let MessageBalance { mut retryable, mut non_retryable, - } = *storage.get(&key)?.unwrap_or_default(); - + } = *current_balance; if message.has_retryable_amount() { - retryable.saturating_add(message.amount() as u128); + retryable = retryable.saturating_add(message.amount() as u128); } else { - non_retryable.saturating_add(message.amount() as u128); + non_retryable = non_retryable.saturating_add(message.amount() as u128); } - let new_balance = MessageBalance { retryable, non_retryable, @@ -130,15 +129,15 @@ where Some(new_amount) => { let storage = block_st_transaction.storage::(); let new_balance = MessageBalance { - retryable: new_amount, - non_retryable, + retryable, + non_retryable: new_amount, }; return Ok(storage.insert(&key, &new_balance)?); } None => { return Err(IndexationError::MessageBalanceWouldUnderflow { owner: message.recipient().clone(), - current_amount: retryable, + current_amount: non_retryable, requested_deduction: message.amount() as u128, retryable: false, }); @@ -156,11 +155,11 @@ where { let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); let storage = block_st_transaction.storage::(); - let mut amount = *storage.get(&key)?.unwrap_or_default(); - amount.saturating_add(coin.amount as u128); + let current_amount = *storage.get(&key)?.unwrap_or_default(); + let new_amount = current_amount.saturating_add(coin.amount as u128); let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &amount)?) + Ok(storage.insert(&key, &new_amount)?) } fn decrease_coin_balance( diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index a0576bc22c7..c7559b6cd21 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -57,7 +57,7 @@ impl TableWithBlueprint for CoinBalances { } } -#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] pub struct MessageBalance { pub retryable: TotalBalanceAmount, pub non_retryable: TotalBalanceAmount, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 8a3abdb7059..d50ba23c139 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -265,7 +265,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { balances .entry(*base_asset_id) .and_modify(|current| { - current.saturating_add(non_retryable); + *current = current.saturating_add(non_retryable); }) .or_insert(non_retryable); } From b2fc999368426ceb1177ff96826de827e721dba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 20 Nov 2024 10:13:34 +0100 Subject: [PATCH 097/111] Add UTs for the indexation module --- .../fuel-core/src/graphql_api/indexation.rs | 520 +++++++++++++++++- .../src/graphql_api/storage/balances.rs | 2 +- 2 files changed, 520 insertions(+), 2 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index a3df0c28889..891e9385eac 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -28,7 +28,7 @@ use super::{ }, }; -#[derive(derive_more::From, derive_more::Display)] +#[derive(derive_more::From, derive_more::Display, Debug)] pub enum IndexationError { #[display( fmt = "Coin balance would underflow for owner: {}, asset_id: {}, current_amount: {}, requested_deduction: {}", @@ -216,3 +216,521 @@ where } => Ok(()), } } + +#[cfg(test)] +mod tests { + use fuel_core_chain_config::{ + CoinConfig, + CoinConfigGenerator, + }; + use fuel_core_storage::{ + transactional::WriteTransaction, + StorageAsMut, + }; + use fuel_core_types::{ + entities::{ + coins::coin::Coin, + relayer::message::MessageV1, + Message, + }, + fuel_tx::{ + Address, + AssetId, + }, + services::executor::Event, + }; + + use crate::{ + database::{ + database_description::off_chain::OffChain, + Database, + }, + graphql_api::{ + indexation::{ + process_balances_update, + IndexationError, + }, + ports::worker::OffChainDatabaseTransaction, + storage::balances::{ + CoinBalances, + CoinBalancesKey, + MessageBalance, + MessageBalances, + }, + }, + }; + + impl PartialEq for IndexationError { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::CoinBalanceWouldUnderflow { + owner: l_owner, + asset_id: l_asset_id, + current_amount: l_current_amount, + requested_deduction: l_requested_deduction, + }, + Self::CoinBalanceWouldUnderflow { + owner: r_owner, + asset_id: r_asset_id, + current_amount: r_current_amount, + requested_deduction: r_requested_deduction, + }, + ) => { + l_owner == r_owner + && l_asset_id == r_asset_id + && l_current_amount == r_current_amount + && l_requested_deduction == r_requested_deduction + } + ( + Self::MessageBalanceWouldUnderflow { + owner: l_owner, + current_amount: l_current_amount, + requested_deduction: l_requested_deduction, + retryable: l_retryable, + }, + Self::MessageBalanceWouldUnderflow { + owner: r_owner, + current_amount: r_current_amount, + requested_deduction: r_requested_deduction, + retryable: r_retryable, + }, + ) => { + l_owner == r_owner + && l_current_amount == r_current_amount + && l_requested_deduction == r_requested_deduction + && l_retryable == r_retryable + } + (Self::StorageError(l0), Self::StorageError(r0)) => l0 == r0, + _ => false, + } + } + } + + fn make_coin(owner: &Address, asset_id: &AssetId, amount: u64) -> Coin { + Coin { + utxo_id: Default::default(), + owner: owner.clone(), + amount, + asset_id: asset_id.clone(), + tx_pointer: Default::default(), + } + } + + fn make_retryable_message(owner: &Address, amount: u64) -> Message { + Message::V1(MessageV1 { + sender: Default::default(), + recipient: owner.clone(), + nonce: Default::default(), + amount, + data: vec![1], + da_height: Default::default(), + }) + } + + fn make_nonretryable_message(owner: &Address, amount: u64) -> Message { + let mut message = make_retryable_message(owner, amount); + message.set_data(vec![]); + message + } + + fn assert_coin_balance( + tx: &mut T, + owner: Address, + asset_id: AssetId, + expected_balance: u128, + ) where + T: OffChainDatabaseTransaction, + { + let key = CoinBalancesKey::new(&owner, &asset_id); + let balance = tx + .storage::() + .get(&key) + .expect("should correctly query db") + .expect("should have balance"); + + assert_eq!(*balance, expected_balance); + } + + fn assert_message_balance( + tx: &mut T, + owner: Address, + expected_balance: MessageBalance, + ) where + T: OffChainDatabaseTransaction, + { + let balance = tx + .storage::() + .get(&owner) + .expect("should correctly query db") + .expect("should have balance"); + + assert_eq!(*balance, expected_balance); + } + + #[test] + fn balances_enabled_flag_is_respected() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_DISABLED: bool = false; + + let owner_1 = Address::from([1; 32]); + let owner_2 = Address::from([2; 32]); + + let asset_id_1 = AssetId::from([11; 32]); + let asset_id_2 = AssetId::from([12; 32]); + + // Initial set of coins + let events: Vec = vec![ + Event::CoinCreated(make_coin(&owner_1, &asset_id_1, 100)), + Event::CoinConsumed(make_coin(&owner_1, &asset_id_2, 200)), + Event::MessageImported(make_retryable_message(&owner_1, 300)), + Event::MessageConsumed(make_nonretryable_message(&owner_2, 400)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_DISABLED) + .expect("should process balance"); + }); + + let key = CoinBalancesKey::new(&owner_1, &asset_id_1); + let balance = tx + .storage::() + .get(&key) + .expect("should correctly query db"); + assert!(balance.is_none()); + + let key = CoinBalancesKey::new(&owner_1, &asset_id_2); + let balance = tx + .storage::() + .get(&key) + .expect("should correctly query db"); + assert!(balance.is_none()); + + let balance = tx + .storage::() + .get(&owner_1) + .expect("should correctly query db"); + assert!(balance.is_none()); + + let balance = tx + .storage::() + .get(&owner_2) + .expect("should correctly query db"); + assert!(balance.is_none()); + } + + #[test] + fn coins() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_ENABLED: bool = true; + + let owner_1 = Address::from([1; 32]); + let owner_2 = Address::from([2; 32]); + + let asset_id_1 = AssetId::from([11; 32]); + let asset_id_2 = AssetId::from([12; 32]); + + // Initial set of coins + let events: Vec = vec![ + Event::CoinCreated(make_coin(&owner_1, &asset_id_1, 100)), + Event::CoinCreated(make_coin(&owner_1, &asset_id_2, 200)), + Event::CoinCreated(make_coin(&owner_2, &asset_id_1, 300)), + Event::CoinCreated(make_coin(&owner_2, &asset_id_2, 400)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_coin_balance(&mut tx, owner_1, asset_id_1, 100); + assert_coin_balance(&mut tx, owner_1, asset_id_2, 200); + assert_coin_balance(&mut tx, owner_2, asset_id_1, 300); + assert_coin_balance(&mut tx, owner_2, asset_id_2, 400); + + // Add some more coins + let events: Vec = vec![ + Event::CoinCreated(make_coin(&owner_1, &asset_id_1, 1)), + Event::CoinCreated(make_coin(&owner_1, &asset_id_2, 2)), + Event::CoinCreated(make_coin(&owner_2, &asset_id_1, 3)), + Event::CoinCreated(make_coin(&owner_2, &asset_id_2, 4)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_coin_balance(&mut tx, owner_1, asset_id_1, 101); + assert_coin_balance(&mut tx, owner_1, asset_id_2, 202); + assert_coin_balance(&mut tx, owner_2, asset_id_1, 303); + assert_coin_balance(&mut tx, owner_2, asset_id_2, 404); + + // Consume some coins + let events: Vec = vec![ + Event::CoinConsumed(make_coin(&owner_1, &asset_id_1, 100)), + Event::CoinConsumed(make_coin(&owner_1, &asset_id_2, 200)), + Event::CoinConsumed(make_coin(&owner_2, &asset_id_1, 300)), + Event::CoinConsumed(make_coin(&owner_2, &asset_id_2, 400)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_coin_balance(&mut tx, owner_1, asset_id_1, 1); + assert_coin_balance(&mut tx, owner_1, asset_id_2, 2); + assert_coin_balance(&mut tx, owner_2, asset_id_1, 3); + assert_coin_balance(&mut tx, owner_2, asset_id_2, 4); + } + + #[test] + fn messages() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_ENABLED: bool = true; + + let owner_1 = Address::from([1; 32]); + let owner_2 = Address::from([2; 32]); + + // Initial set of messages + let events: Vec = vec![ + Event::MessageImported(make_retryable_message(&owner_1, 100)), + Event::MessageImported(make_retryable_message(&owner_2, 200)), + Event::MessageImported(make_nonretryable_message(&owner_1, 300)), + Event::MessageImported(make_nonretryable_message(&owner_2, 400)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_message_balance( + &mut tx, + owner_1, + MessageBalance { + retryable: 100, + non_retryable: 300, + }, + ); + + assert_message_balance( + &mut tx, + owner_2, + MessageBalance { + retryable: 200, + non_retryable: 400, + }, + ); + + // Add some messages + let events: Vec = vec![ + Event::MessageImported(make_retryable_message(&owner_1, 1)), + Event::MessageImported(make_retryable_message(&owner_2, 2)), + Event::MessageImported(make_nonretryable_message(&owner_1, 3)), + Event::MessageImported(make_nonretryable_message(&owner_2, 4)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_message_balance( + &mut tx, + owner_1, + MessageBalance { + retryable: 101, + non_retryable: 303, + }, + ); + + assert_message_balance( + &mut tx, + owner_2, + MessageBalance { + retryable: 202, + non_retryable: 404, + }, + ); + + // Consume some messages + let events: Vec = vec![ + Event::MessageConsumed(make_retryable_message(&owner_1, 100)), + Event::MessageConsumed(make_retryable_message(&owner_2, 200)), + Event::MessageConsumed(make_nonretryable_message(&owner_1, 300)), + Event::MessageConsumed(make_nonretryable_message(&owner_2, 400)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_message_balance( + &mut tx, + owner_1, + MessageBalance { + retryable: 1, + non_retryable: 3, + }, + ); + + assert_message_balance( + &mut tx, + owner_2, + MessageBalance { + retryable: 2, + non_retryable: 4, + }, + ); + } + + #[test] + fn coin_balance_overflow_does_not_error() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_ENABLED: bool = true; + + let owner = Address::from([1; 32]); + let asset_id = AssetId::from([11; 32]); + + // Make the initial balance huge + let key = CoinBalancesKey::new(&owner, &asset_id); + let balance = tx + .storage::() + .insert(&key, &u128::MAX) + .expect("should correctly query db"); + + assert_coin_balance(&mut tx, owner, asset_id, u128::MAX); + + // Try to add more coins + let events: Vec = + vec![Event::CoinCreated(make_coin(&owner, &asset_id, 1))]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_coin_balance(&mut tx, owner, asset_id, u128::MAX); + } + + #[test] + fn message_balance_overflow_does_not_error() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_ENABLED: bool = true; + const MAX_BALANCES: MessageBalance = MessageBalance { + retryable: u128::MAX, + non_retryable: u128::MAX, + }; + + let owner = Address::from([1; 32]); + + // Make the initial balance huge + let balance = tx + .storage::() + .insert(&owner, &MAX_BALANCES) + .expect("should correctly query db"); + + assert_message_balance(&mut tx, owner, MAX_BALANCES); + + // Try to add more coins + let events: Vec = vec![ + Event::MessageImported(make_retryable_message(&owner, 1)), + Event::MessageImported(make_nonretryable_message(&owner, 1)), + ]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + assert_message_balance(&mut tx, owner, MAX_BALANCES); + } + + #[test] + fn coin_balance_underflow_causes_error() { + use tempfile::TempDir; + let tmp_dir = TempDir::new().unwrap(); + let mut db: Database = + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); + let mut tx = db.write_transaction(); + + const BALANCES_ARE_ENABLED: bool = true; + + let owner = Address::from([1; 32]); + let asset_id_1 = AssetId::from([11; 32]); + let asset_id_2 = AssetId::from([12; 32]); + + // Initial set of coins + let events: Vec = + vec![Event::CoinCreated(make_coin(&owner, &asset_id_1, 100))]; + + events.iter().for_each(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED) + .expect("should process balance"); + }); + + // Consume more coins than available + let events: Vec = vec![ + Event::CoinConsumed(make_coin(&owner, &asset_id_1, 10000)), + Event::CoinConsumed(make_coin(&owner, &asset_id_2, 20000)), + ]; + + let expected_errors = vec![ + IndexationError::CoinBalanceWouldUnderflow { + owner: owner.clone(), + asset_id: asset_id_1.clone(), + current_amount: 100, + requested_deduction: 10000, + }, + IndexationError::CoinBalanceWouldUnderflow { + owner: owner.clone(), + asset_id: asset_id_2.clone(), + current_amount: 0, + requested_deduction: 20000, + }, + ]; + + let actual_errors: Vec<_> = events + .iter() + .map(|event| { + process_balances_update(event, &mut tx, BALANCES_ARE_ENABLED).unwrap_err() + }) + .collect(); + + assert_eq!(expected_errors, actual_errors); + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index c7559b6cd21..2d32af9afc0 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -57,7 +57,7 @@ impl TableWithBlueprint for CoinBalances { } } -#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] pub struct MessageBalance { pub retryable: TotalBalanceAmount, pub non_retryable: TotalBalanceAmount, From b7211034491fa667a7a143a1b6dda4dba276278e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 20 Nov 2024 10:53:30 +0100 Subject: [PATCH 098/111] Simplify implementation of balances indexation --- .../fuel-core/src/graphql_api/indexation.rs | 81 ++++++++----------- 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index 891e9385eac..84d9d67c747 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -60,7 +60,6 @@ pub enum IndexationError { StorageError(StorageError), } -// TODO[RC]: A lot of duplication below, consider refactoring. fn increase_message_balance( block_st_transaction: &mut T, message: &Message, @@ -102,48 +101,35 @@ where mut retryable, mut non_retryable, } = *storage.get(&key)?.unwrap_or_default(); + let current_balance = if message.has_retryable_amount() { + retryable + } else { + non_retryable + }; - if message.has_retryable_amount() { - let maybe_new_amount = retryable.checked_sub(message.amount() as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - let new_balance = MessageBalance { + current_balance + .checked_sub(message.amount() as u128) + .ok_or_else(|| IndexationError::MessageBalanceWouldUnderflow { + owner: message.recipient().clone(), + current_amount: current_balance, + requested_deduction: message.amount() as u128, + retryable: message.has_retryable_amount(), + }) + .and_then(|new_amount| { + let storage = block_st_transaction.storage::(); + let new_balance = if message.has_retryable_amount() { + MessageBalance { retryable: new_amount, non_retryable, - }; - return Ok(storage.insert(&key, &new_balance)?); - } - None => { - return Err(IndexationError::MessageBalanceWouldUnderflow { - owner: message.recipient().clone(), - current_amount: retryable, - requested_deduction: message.amount() as u128, - retryable: true, - }); - } - } - } else { - let maybe_new_amount = non_retryable.checked_sub(message.amount() as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - let new_balance = MessageBalance { + } + } else { + MessageBalance { retryable, non_retryable: new_amount, - }; - return Ok(storage.insert(&key, &new_balance)?); - } - None => { - return Err(IndexationError::MessageBalanceWouldUnderflow { - owner: message.recipient().clone(), - current_amount: non_retryable, - requested_deduction: message.amount() as u128, - retryable: false, - }); - } - } - } + } + }; + storage.insert(&key, &new_balance).map_err(Into::into) + }) } fn increase_coin_balance( @@ -173,19 +159,20 @@ where let storage = block_st_transaction.storage::(); let mut current_amount = *storage.get(&key)?.unwrap_or_default(); - let maybe_new_amount = current_amount.checked_sub(coin.amount as u128); - match maybe_new_amount { - Some(new_amount) => { - let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &new_amount)?) - } - None => Err(IndexationError::CoinBalanceWouldUnderflow { + current_amount + .checked_sub(coin.amount as u128) + .ok_or_else(|| IndexationError::CoinBalanceWouldUnderflow { owner: coin.owner.clone(), asset_id: coin.asset_id.clone(), current_amount, requested_deduction: coin.amount as u128, - }), - } + }) + .and_then(|new_amount| { + block_st_transaction + .storage::() + .insert(&key, &new_amount) + .map_err(Into::into) + }) } pub(crate) fn process_balances_update( From bd16f6c6d75d7c5c1fc9eea5aef41040ace4e776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 20 Nov 2024 12:23:09 +0100 Subject: [PATCH 099/111] Satisfy Clippy --- .../fuel-core/src/graphql_api/indexation.rs | 54 +++++++------------ .../src/graphql_api/worker_service.rs | 32 ++--------- .../service/adapters/graphql_api/off_chain.rs | 2 +- 3 files changed, 26 insertions(+), 62 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index 84d9d67c747..fde469e9100 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -1,10 +1,6 @@ use fuel_core_storage::{ Error as StorageError, - Mappable, - Result as StorageResult, StorageAsMut, - StorageInspect, - StorageMutate, }; use fuel_core_types::{ entities::{ @@ -69,7 +65,7 @@ where { let key = message.recipient(); let storage = block_st_transaction.storage::(); - let current_balance = storage.get(&key)?.unwrap_or_default(); + let current_balance = storage.get(key)?.unwrap_or_default(); let MessageBalance { mut retryable, mut non_retryable, @@ -85,7 +81,7 @@ where }; let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &new_balance)?) + Ok(storage.insert(key, &new_balance)?) } fn decrease_message_balance( @@ -98,9 +94,9 @@ where let key = message.recipient(); let storage = block_st_transaction.storage::(); let MessageBalance { - mut retryable, - mut non_retryable, - } = *storage.get(&key)?.unwrap_or_default(); + retryable, + non_retryable, + } = *storage.get(key)?.unwrap_or_default(); let current_balance = if message.has_retryable_amount() { retryable } else { @@ -110,7 +106,7 @@ where current_balance .checked_sub(message.amount() as u128) .ok_or_else(|| IndexationError::MessageBalanceWouldUnderflow { - owner: message.recipient().clone(), + owner: *message.recipient(), current_amount: current_balance, requested_deduction: message.amount() as u128, retryable: message.has_retryable_amount(), @@ -128,7 +124,7 @@ where non_retryable: new_amount, } }; - storage.insert(&key, &new_balance).map_err(Into::into) + storage.insert(key, &new_balance).map_err(Into::into) }) } @@ -157,13 +153,13 @@ where { let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); let storage = block_st_transaction.storage::(); - let mut current_amount = *storage.get(&key)?.unwrap_or_default(); + let current_amount = *storage.get(&key)?.unwrap_or_default(); current_amount .checked_sub(coin.amount as u128) .ok_or_else(|| IndexationError::CoinBalanceWouldUnderflow { - owner: coin.owner.clone(), - asset_id: coin.asset_id.clone(), + owner: coin.owner, + asset_id: coin.asset_id, current_amount, requested_deduction: coin.amount as u128, }) @@ -196,20 +192,12 @@ where } Event::CoinCreated(coin) => increase_coin_balance(block_st_transaction, coin), Event::CoinConsumed(coin) => decrease_coin_balance(block_st_transaction, coin), - Event::ForcedTransactionFailed { - id, - block_height, - failure, - } => Ok(()), + Event::ForcedTransactionFailed { .. } => Ok(()), } } #[cfg(test)] mod tests { - use fuel_core_chain_config::{ - CoinConfig, - CoinConfigGenerator, - }; use fuel_core_storage::{ transactional::WriteTransaction, StorageAsMut, @@ -297,9 +285,9 @@ mod tests { fn make_coin(owner: &Address, asset_id: &AssetId, amount: u64) -> Coin { Coin { utxo_id: Default::default(), - owner: owner.clone(), + owner: *owner, amount, - asset_id: asset_id.clone(), + asset_id: *asset_id, tx_pointer: Default::default(), } } @@ -307,7 +295,7 @@ mod tests { fn make_retryable_message(owner: &Address, amount: u64) -> Message { Message::V1(MessageV1 { sender: Default::default(), - recipient: owner.clone(), + recipient: *owner, nonce: Default::default(), amount, data: vec![1], @@ -608,8 +596,7 @@ mod tests { // Make the initial balance huge let key = CoinBalancesKey::new(&owner, &asset_id); - let balance = tx - .storage::() + tx.storage::() .insert(&key, &u128::MAX) .expect("should correctly query db"); @@ -645,8 +632,7 @@ mod tests { let owner = Address::from([1; 32]); // Make the initial balance huge - let balance = tx - .storage::() + tx.storage::() .insert(&owner, &MAX_BALANCES) .expect("should correctly query db"); @@ -698,14 +684,14 @@ mod tests { let expected_errors = vec![ IndexationError::CoinBalanceWouldUnderflow { - owner: owner.clone(), - asset_id: asset_id_1.clone(), + owner, + asset_id: asset_id_1, current_amount: 100, requested_deduction: 10000, }, IndexationError::CoinBalanceWouldUnderflow { - owner: owner.clone(), - asset_id: asset_id_2.clone(), + owner, + asset_id: asset_id_2, current_amount: 0, requested_deduction: 20000, }, diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index df1c4618897..8ab8e2492ea 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -3,18 +3,10 @@ use self::indexation::IndexationError; use super::{ da_compression::da_compress_block, indexation, - storage::{ - balances::{ - CoinBalancesKey, - ItemAmount, - MessageBalance, - TotalBalanceAmount, - }, - old::{ - OldFuelBlockConsensus, - OldFuelBlocks, - OldTransactions, - }, + storage::old::{ + OldFuelBlockConsensus, + OldFuelBlocks, + OldTransactions, }, }; use crate::{ @@ -24,10 +16,6 @@ use crate::{ worker::OffChainDatabaseTransaction, }, storage::{ - balances::{ - CoinBalances, - MessageBalances, - }, blocks::FuelBlockIdsToHeights, coins::{ owner_coin_id_key, @@ -54,11 +42,8 @@ use fuel_core_services::{ }; use fuel_core_storage::{ Error as StorageError, - Mappable, Result as StorageResult, StorageAsMut, - StorageInspect, - StorageMutate, }; use fuel_core_types::{ blockchain::{ @@ -68,11 +53,7 @@ use fuel_core_types::{ }, consensus::Consensus, }, - entities::{ - coins::coin::Coin, - relayer::transaction::RelayedTransactionStatus, - Message, - }, + entities::relayer::transaction::RelayedTransactionStatus, fuel_tx::{ field::{ Inputs, @@ -84,8 +65,6 @@ use fuel_core_types::{ CoinPredicate, CoinSigned, }, - Address, - AssetId, Contract, Input, Output, @@ -119,7 +98,6 @@ use std::{ ops::Deref, }; use tracing::{ - debug, error, info, }; diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index d50ba23c139..70dcfcb6a9e 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -254,7 +254,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { .unwrap_or_default() .into_owned() as TotalBalanceAmount; - balances.insert(asset_id.clone(), coins); + balances.insert(*asset_id, coins); } if let Some(messages) = self.storage_as_ref::().get(owner)? { From 53631e7143e04c12b9e81b958624e8173519d080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 21 Nov 2024 12:04:36 +0100 Subject: [PATCH 100/111] Extract and reuse `BALANCES_INDEXATION_ENABLED` flag --- .../fuel-core/src/service/genesis/importer/off_chain.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 3b7607c738c..745b6b92a96 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -43,6 +43,9 @@ use super::{ Handler, }; +// We always want to enable balances indexation if we're starting at genesis. +const BALANCES_INDEXATION_ENABLED: bool = true; + impl ImportTable for Handler { type TableInSnapshot = TransactionStatuses; type TableBeingWritten = TransactionStatuses; @@ -107,9 +110,6 @@ impl ImportTable for Handler { group: Vec>, tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { - // We always want to enable balances indexation if we're starting at genesis. - const BALANCES_INDEXATION_ENABLED: bool = true; - let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); @@ -128,9 +128,6 @@ impl ImportTable for Handler { group: Vec>, tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { - // We always want to enable balances indexation if we're starting at genesis. - const BALANCES_INDEXATION_ENABLED: bool = true; - let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); From a8b950b589f61c463f79c88635b6acf8fe8ad5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 21 Nov 2024 13:59:46 +0100 Subject: [PATCH 101/111] Use saturating_add when calculatin balances (hard to overflow u128 with u64s) --- crates/fuel-core/src/query/balance.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 4c3a5cc9e84..554cf4ddf91 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -118,16 +118,8 @@ impl ReadView { let amount: &mut TotalBalanceAmount = amounts_per_asset .entry(*coin.asset_id(base_asset_id)) .or_default(); - let new_amount = amount - .checked_add(coin.amount() as TotalBalanceAmount) - .unwrap_or_else(|| { - // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 - error!( - asset_id=%coin.asset_id(base_asset_id), - prev_balance=%amount, - "unable to change balance due to overflow"); - u128::MAX - }); + let new_amount = + amount.saturating_add(coin.amount() as TotalBalanceAmount); *amount = new_amount; Ok(amounts_per_asset) }, From 2e6ade7b7d3db56386d4d204fa2c5a9f25f89a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 21 Nov 2024 14:06:15 +0100 Subject: [PATCH 102/111] Use saturating_add when calculating balances (hard to overflow u128 with u64s) --- crates/fuel-core/src/query/balance.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 554cf4ddf91..85ad1b908d7 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -61,14 +61,7 @@ impl ReadView { .coins() .map(|res| res.map(|coins| coins.amount())) .try_fold(0u128, |balance, amount| async move { - Ok(balance.checked_add(amount as u128).unwrap_or_else(|| { - // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 - error!( - %asset_id, - prev_balance=%balance, - "unable to change balance due to overflow"); - u128::MAX - })) + Ok(balance.saturating_add(amount as TotalBalanceAmount)) }) .await? as TotalBalanceAmount }; From 7a955064010704e7801dc0eda47ee8f48bd1d162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 21 Nov 2024 14:16:55 +0100 Subject: [PATCH 103/111] Do not import `tracing` and remove some log messages --- crates/fuel-core/src/graphql_api/worker_service.rs | 8 ++------ crates/fuel-core/src/query/balance.rs | 8 -------- crates/fuel-core/src/service.rs | 3 +-- .../src/service/adapters/graphql_api/off_chain.rs | 4 ---- 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 8ab8e2492ea..362b1a39bc4 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -97,10 +97,6 @@ use std::{ borrow::Cow, ops::Deref, }; -use tracing::{ - error, - info, -}; #[cfg(test)] mod tests; @@ -213,7 +209,7 @@ where Err(err @ IndexationError::CoinBalanceWouldUnderflow { .. }) | Err(err @ IndexationError::MessageBalanceWouldUnderflow { .. }) => { // TODO[RC]: Balances overflow to be correctly handled. See: https://github.com/FuelLabs/fuel-core/issues/2428 - error!("Balances underflow detected: {}", err); + tracing::error!("Balances underflow detected: {}", err); } } @@ -500,7 +496,7 @@ where } let balances_enabled = self.off_chain_database.balances_enabled()?; - info!("Balances cache available: {}", balances_enabled); + tracing::info!("Balances cache available: {}", balances_enabled); let InitializeTask { chain_id, diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 85ad1b908d7..9d793b4ec5c 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -32,10 +32,6 @@ use futures::{ StreamExt, TryStreamExt, }; -use tracing::{ - debug, - error, -}; pub mod asset_query; @@ -47,10 +43,8 @@ impl ReadView { base_asset_id: AssetId, ) -> StorageResult { let amount = if self.balances_enabled { - debug!(%owner, %asset_id, "Querying balance with balances cache"); self.off_chain.balance(&owner, &asset_id, &base_asset_id)? } else { - debug!(%owner, %asset_id, "Querying balance without balances cache"); AssetQuery::new( &owner, &AssetSpendTarget::new(asset_id, u64::MAX, u16::MAX), @@ -100,7 +94,6 @@ impl ReadView { base_asset_id: &'a AssetId, direction: IterDirection, ) -> impl Stream> + 'a { - debug!(%owner, "Querying balances without balances cache"); let query = AssetsQuery::new(owner, None, None, self, base_asset_id); let stream = query.coins(); @@ -153,7 +146,6 @@ impl ReadView { base_asset_id: &AssetId, direction: IterDirection, ) -> impl Stream> + 'a { - debug!(%owner, "Querying balances using balances cache"); match self.off_chain.balances(owner, base_asset_id) { Ok(balances) => { let iter = if direction == IterDirection::Reverse { diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 9474d08601f..fdd20659030 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -52,7 +52,6 @@ use std::{ net::SocketAddr, sync::Arc, }; -use tracing::info; pub use config::{ Config, @@ -226,7 +225,7 @@ impl FuelService { let on_chain_view = database.on_chain().latest_view()?; if on_chain_view.get_genesis().is_err() { let all_indexations = IndexationKind::all().collect(); - info!( + tracing::info!( "No genesis, initializing metadata with all supported indexations: {:?}", all_indexations ); diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 70dcfcb6a9e..bb8770ee219 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -80,7 +80,6 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; -use tracing::debug; impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { @@ -229,11 +228,8 @@ impl OffChainDatabase for OffChainIterableKeyValueView { let total = coins.checked_add(non_retryable).ok_or(anyhow::anyhow!( "Total balance overflow: coins: {coins}, messages: {non_retryable}" ))?; - - debug!(%coins, %non_retryable, total, "total balance"); Ok(total) } else { - debug!(%coins, "total balance"); Ok(coins) } } From 33f75af6d4238a76f4be4b6fc1c0c119b0d53d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Thu, 21 Nov 2024 15:59:32 +0100 Subject: [PATCH 104/111] Move the indexation initialization to combined databases --- crates/fuel-core/src/combined_database.rs | 45 ++++++++++++++++++++++- crates/fuel-core/src/service.rs | 43 +--------------------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 8696a8b3969..2f52c86938f 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -7,7 +7,11 @@ use crate::{ off_chain::OffChain, on_chain::OnChain, relayer::Relayer, + DatabaseDescription, + DatabaseMetadata, + IndexationKind, }, + metadata::MetadataTable, Database, GenesisDatabase, Result as DatabaseResult, @@ -28,7 +32,12 @@ use fuel_core_storage::tables::{ ContractsState, Messages, }; -use fuel_core_storage::Result as StorageResult; +use fuel_core_storage::{ + transactional::ReadTransaction, + Result as StorageResult, + StorageAsMut, +}; +use fuel_core_txpool::ports::AtomicView; use fuel_core_types::fuel_types::BlockHeight; use std::path::PathBuf; @@ -154,6 +163,40 @@ impl CombinedDatabase { Ok(()) } + pub fn initialize(&self) -> StorageResult<()> { + self.initialize_indexation()?; + Ok(()) + } + + fn initialize_indexation(&self) -> StorageResult<()> { + // When genesis is missing write to the database that balances cache should be used. + let on_chain_view = self.on_chain().latest_view()?; + if on_chain_view.get_genesis().is_err() { + let all_indexations = IndexationKind::all().collect(); + tracing::info!( + "No genesis, initializing metadata with all supported indexations: {:?}", + all_indexations + ); + let off_chain_view = self.off_chain().latest_view()?; + let mut database_tx = off_chain_view.read_transaction(); + database_tx + .storage_as_mut::>() + .insert( + &(), + &DatabaseMetadata::V2 { + version: ::version(), + height: Default::default(), + indexation_availability: all_indexations, + }, + )?; + self.off_chain() + .data + .commit_changes(None, database_tx.into_changes())?; + }; + + Ok(()) + } + pub fn on_chain(&self) -> &Database { &self.on_chain } diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index fdd20659030..5365b245d1d 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -4,16 +4,7 @@ use crate::{ CombinedDatabase, ShutdownListener, }, - database::{ - database_description::{ - off_chain::OffChain, - DatabaseDescription, - DatabaseMetadata, - IndexationKind, - }, - metadata::MetadataTable, - Database, - }, + database::Database, service::{ adapters::{ ExecutorAdapter, @@ -134,6 +125,7 @@ impl FuelService { // initialize state tracing::info!("Initializing database"); database.check_version()?; + database.initialize()?; Self::make_database_compatible_with_config( &mut database, @@ -141,8 +133,6 @@ impl FuelService { shutdown_listener, )?; - Self::write_metadata_at_genesis(&database)?; - // initialize sub services tracing::info!("Initializing sub services"); database.sync_aux_db_heights(shutdown_listener)?; @@ -220,35 +210,6 @@ impl FuelService { Ok(()) } - // When genesis is missing write to the database that balances cache should be used. - fn write_metadata_at_genesis(database: &CombinedDatabase) -> anyhow::Result<()> { - let on_chain_view = database.on_chain().latest_view()?; - if on_chain_view.get_genesis().is_err() { - let all_indexations = IndexationKind::all().collect(); - tracing::info!( - "No genesis, initializing metadata with all supported indexations: {:?}", - all_indexations - ); - let off_chain_view = database.off_chain().latest_view()?; - let mut database_tx = off_chain_view.read_transaction(); - database_tx - .storage_as_mut::>() - .insert( - &(), - &DatabaseMetadata::V2 { - version: ::version(), - height: Default::default(), - indexation_availability: all_indexations, - }, - )?; - database - .off_chain() - .data - .commit_changes(None, database_tx.into_changes())?; - } - Ok(()) - } - fn make_database_compatible_with_config( combined_database: &mut CombinedDatabase, config: &Config, From 6b8f18ebf6da15b2fa366a8d06fc46e5164c40b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 09:09:35 +0100 Subject: [PATCH 105/111] Return balances via iterator, not collection --- crates/fuel-core/src/graphql_api/ports.rs | 8 +- crates/fuel-core/src/query/balance.rs | 27 ++---- .../service/adapters/graphql_api/off_chain.rs | 89 +++++++++++++------ 3 files changed, 74 insertions(+), 50 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 50d14a1f5d5..0545324d4dc 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -62,10 +62,7 @@ use fuel_core_types::{ }, tai64::Tai64, }; -use std::{ - collections::BTreeMap, - sync::Arc, -}; +use std::sync::Arc; use super::storage::balances::TotalBalanceAmount; @@ -87,7 +84,8 @@ pub trait OffChainDatabase: Send + Sync { &self, owner: &Address, base_asset_id: &AssetId, - ) -> StorageResult>; + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult<(AssetId, TotalBalanceAmount)>>; fn owned_coins_ids( &self, diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 9d793b4ec5c..58e53dafad8 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -1,7 +1,6 @@ use std::{ cmp::Ordering, collections::HashMap, - future, }; use crate::{ @@ -146,24 +145,14 @@ impl ReadView { base_asset_id: &AssetId, direction: IterDirection, ) -> impl Stream> + 'a { - match self.off_chain.balances(owner, base_asset_id) { - Ok(balances) => { - let iter = if direction == IterDirection::Reverse { - itertools::Either::Left(balances.into_iter().rev()) - } else { - itertools::Either::Right(balances.into_iter()) - }; - stream::iter(iter.map(|(asset_id, amount)| AddressBalance { - owner: *owner, - amount, + stream::iter(self.off_chain.balances(owner, base_asset_id, direction)) + .map(move |result| { + result.map(|(asset_id, amount)| AddressBalance { + owner: owner.clone(), asset_id, - })) - .map(Ok) - .into_stream() - .yield_each(self.batch_size) - .left_stream() - } - Err(err) => stream::once(future::ready(Err(err))).right_stream(), - } + amount, + }) + }) + .yield_each(self.batch_size) } } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index bb8770ee219..bafd7a968cf 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::borrow::Cow; use crate::{ database::{ @@ -238,35 +238,72 @@ impl OffChainDatabase for OffChainIterableKeyValueView { &self, owner: &Address, base_asset_id: &AssetId, - ) -> StorageResult> { - let mut balances = BTreeMap::new(); - for balance_key in self.iter_all_by_prefix_keys::(Some(owner)) { - let key = balance_key?; - let asset_id = key.asset_id(); - - let coins = self - .storage_as_ref::() - .get(&key)? - .unwrap_or_default() - .into_owned() as TotalBalanceAmount; + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult<(AssetId, TotalBalanceAmount)>> { + let base_asset_id = base_asset_id.clone(); + let (maybe_messages_balance, errors_1) = self + .storage_as_ref::() + .get(owner) + .map_or_else( + |err| (None, std::iter::once(Err(err)).into_boxed()), + |value| { + ( + value.map(|value| value.non_retryable), + std::iter::empty().into_boxed(), + ) + }, + ); - balances.insert(*asset_id, coins); - } + let (maybe_base_coin_balance, errors_2) = self + .storage_as_ref::() + .get(&CoinBalancesKey::new(owner, &base_asset_id)) + .map_or_else( + |err| (None, std::iter::once(Err(err)).into_boxed()), + |value| (value.map(Cow::into_owned), std::iter::empty().into_boxed()), + ); - if let Some(messages) = self.storage_as_ref::().get(owner)? { - let MessageBalance { - retryable: _, - non_retryable, - } = *messages; - balances - .entry(*base_asset_id) - .and_modify(|current| { - *current = current.saturating_add(non_retryable); + let prefix_non_base_asset = owner; + let coins_non_base_asset_iter = self + .iter_all_filtered_keys::( + Some(prefix_non_base_asset), + None, + Some(direction), + ) + .filter_map(move |result| match result { + Ok(key) if *key.asset_id() != base_asset_id => Some(Ok(key)), + Ok(_) => None, + Err(err) => Some(Err(err)), + }) + .map(move |result| { + result.and_then(|key| { + let asset_id = key.asset_id(); + let coin_balance = + self.storage_as_ref::() + .get(&key)? + .unwrap_or_default() + .into_owned() as TotalBalanceAmount; + Ok((*asset_id, coin_balance)) }) - .or_insert(non_retryable); - } + }) + .into_boxed(); - Ok(balances) + errors_1 + .chain(errors_2) + .chain(match (maybe_messages_balance, maybe_base_coin_balance) { + (None, None) => std::iter::empty().into_boxed(), + (None, Some(base_coin_balance)) => { + std::iter::once(Ok((base_asset_id, base_coin_balance))).into_boxed() + } + (Some(messages_balance), None) => { + std::iter::once(Ok((base_asset_id, messages_balance))).into_boxed() + } + (Some(messages_balance), Some(base_coin_balance)) => std::iter::once(Ok( + (base_asset_id, messages_balance + base_coin_balance), + )) + .into_boxed(), + }) + .chain(coins_non_base_asset_iter) + .into_boxed() } } From 2408e568597ab720caa228f036d11eb464533704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 09:16:21 +0100 Subject: [PATCH 106/111] Remove dbg print --- tests/tests/balances.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index 20eb662914c..e5fc2bed243 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -314,6 +314,8 @@ async fn first_5_balances() { assert!(!balances.is_empty()); assert_eq!(balances.len(), 5); + dbg!(&balances); + // Base asset is 3 coins and 2 messages = 50 + 100 + 150 + 60 + 90 assert_eq!(balances[0].asset_id, asset_ids[0]); assert_eq!(balances[0].amount, 450); From 67d17d54c5af69fb62031d8a387f7ef97af82cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 11:37:12 +0100 Subject: [PATCH 107/111] Make the `balances()` implementation more readable --- crates/fuel-core/src/query/balance.rs | 2 +- .../service/adapters/graphql_api/off_chain.rs | 158 +++++++++++++----- tests/tests/balances.rs | 2 - 3 files changed, 113 insertions(+), 49 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 58e53dafad8..dcc589118a3 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -148,7 +148,7 @@ impl ReadView { stream::iter(self.off_chain.balances(owner, base_asset_id, direction)) .map(move |result| { result.map(|(asset_id, amount)| AddressBalance { - owner: owner.clone(), + owner: *owner, asset_id, amount, }) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index bafd7a968cf..2a519b3f2d2 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -34,7 +34,9 @@ use crate::{ OldFuelBlocks, OldTransactions, }, + Column, }, + state::IterableKeyValueView, }; use fuel_core_storage::{ blueprint::BlueprintInspect, @@ -55,6 +57,7 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, StorageAsRef, + StorageRef, }; use fuel_core_types::{ blockchain::{ @@ -240,35 +243,16 @@ impl OffChainDatabase for OffChainIterableKeyValueView { base_asset_id: &AssetId, direction: IterDirection, ) -> BoxedIter<'_, StorageResult<(AssetId, TotalBalanceAmount)>> { - let base_asset_id = base_asset_id.clone(); - let (maybe_messages_balance, errors_1) = self - .storage_as_ref::() - .get(owner) - .map_or_else( - |err| (None, std::iter::once(Err(err)).into_boxed()), - |value| { - ( - value.map(|value| value.non_retryable), - std::iter::empty().into_boxed(), - ) - }, - ); - - let (maybe_base_coin_balance, errors_2) = self - .storage_as_ref::() - .get(&CoinBalancesKey::new(owner, &base_asset_id)) - .map_or_else( - |err| (None, std::iter::once(Err(err)).into_boxed()), - |value| (value.map(Cow::into_owned), std::iter::empty().into_boxed()), - ); - - let prefix_non_base_asset = owner; - let coins_non_base_asset_iter = self - .iter_all_filtered_keys::( - Some(prefix_non_base_asset), - None, - Some(direction), - ) + let base_asset_id = *base_asset_id; + let base_asset_balance = base_asset_balance( + self.storage_as_ref::(), + self.storage_as_ref::(), + owner, + &base_asset_id, + ); + + let non_base_asset_balance = self + .iter_all_filtered_keys::(Some(owner), None, Some(direction)) .filter_map(move |result| match result { Ok(key) if *key.asset_id() != base_asset_id => Some(Ok(key)), Ok(_) => None, @@ -287,26 +271,108 @@ impl OffChainDatabase for OffChainIterableKeyValueView { }) .into_boxed(); - errors_1 - .chain(errors_2) - .chain(match (maybe_messages_balance, maybe_base_coin_balance) { - (None, None) => std::iter::empty().into_boxed(), - (None, Some(base_coin_balance)) => { - std::iter::once(Ok((base_asset_id, base_coin_balance))).into_boxed() - } - (Some(messages_balance), None) => { - std::iter::once(Ok((base_asset_id, messages_balance))).into_boxed() - } - (Some(messages_balance), Some(base_coin_balance)) => std::iter::once(Ok( - (base_asset_id, messages_balance + base_coin_balance), - )) - .into_boxed(), - }) - .chain(coins_non_base_asset_iter) - .into_boxed() + if direction == IterDirection::Forward { + base_asset_balance + .chain(non_base_asset_balance) + .into_boxed() + } else { + non_base_asset_balance + .chain(base_asset_balance) + .into_boxed() + } } } +struct AssetBalanceWithRetrievalErrors<'a> { + balance: Option, + errors: BoxedIter<'a, Result<(AssetId, u128), StorageError>>, +} + +impl<'a> AssetBalanceWithRetrievalErrors<'a> { + fn new( + balance: Option, + errors: BoxedIter<'a, Result<(AssetId, u128), StorageError>>, + ) -> Self { + Self { balance, errors } + } +} + +fn non_retryable_message_balance<'a>( + storage: StorageRef<'a, IterableKeyValueView, MessageBalances>, + owner: &Address, +) -> AssetBalanceWithRetrievalErrors<'a> { + storage.get(owner).map_or_else( + |err| { + AssetBalanceWithRetrievalErrors::new( + None, + std::iter::once(Err(err)).into_boxed(), + ) + }, + |value| { + AssetBalanceWithRetrievalErrors::new( + value.map(|value| value.non_retryable), + std::iter::empty().into_boxed(), + ) + }, + ) +} + +fn base_asset_coin_balance<'a, 'b>( + storage: StorageRef<'a, IterableKeyValueView, CoinBalances>, + owner: &'b Address, + base_asset_id: &'b AssetId, +) -> AssetBalanceWithRetrievalErrors<'a> { + storage + .get(&CoinBalancesKey::new(owner, base_asset_id)) + .map_or_else( + |err| { + AssetBalanceWithRetrievalErrors::new( + None, + std::iter::once(Err(err)).into_boxed(), + ) + }, + |value| { + AssetBalanceWithRetrievalErrors::new( + value.map(Cow::into_owned), + std::iter::empty().into_boxed(), + ) + }, + ) +} + +fn base_asset_balance<'a, 'b>( + messages_storage: StorageRef<'a, IterableKeyValueView, MessageBalances>, + coins_storage: StorageRef<'a, IterableKeyValueView, CoinBalances>, + owner: &'b Address, + base_asset_id: &'b AssetId, +) -> BoxedIter<'a, Result<(AssetId, TotalBalanceAmount), StorageError>> { + let AssetBalanceWithRetrievalErrors { + balance: messages_balance, + errors: message_errors, + } = non_retryable_message_balance(messages_storage, owner); + + let AssetBalanceWithRetrievalErrors { + balance: base_coin_balance, + errors: coin_errors, + } = base_asset_coin_balance(coins_storage, owner, base_asset_id); + + let base_asset_id = *base_asset_id; + let balance = match (messages_balance, base_coin_balance) { + (None, None) => None, + (None, Some(balance)) | (Some(balance), None) => Some(balance), + (Some(msg_balance), Some(coin_balance)) => { + Some(msg_balance.saturating_add(coin_balance)) + } + } + .into_iter() + .map(move |balance| Ok((base_asset_id, balance))); + + message_errors + .chain(coin_errors) + .chain(balance) + .into_boxed() +} + impl worker::OffChainDatabase for Database { type Transaction<'a> = StorageTransaction<&'a mut Self> where Self: 'a; diff --git a/tests/tests/balances.rs b/tests/tests/balances.rs index e5fc2bed243..20eb662914c 100644 --- a/tests/tests/balances.rs +++ b/tests/tests/balances.rs @@ -314,8 +314,6 @@ async fn first_5_balances() { assert!(!balances.is_empty()); assert_eq!(balances.len(), 5); - dbg!(&balances); - // Base asset is 3 coins and 2 messages = 50 + 100 + 150 + 60 + 90 assert_eq!(balances[0].asset_id, asset_ids[0]); assert_eq!(balances[0].amount, 450); From bb2537ec0ee0c771dc8ca4a73aafe76027913a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 12:12:25 +0100 Subject: [PATCH 108/111] Update comment to mention follow-up issue --- crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 2a519b3f2d2..7acd5d47f74 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -220,7 +220,7 @@ impl OffChainDatabase for OffChainIterableKeyValueView { if base_asset_id == asset_id { let MessageBalance { - retryable: _, // TODO[RC]: Handle this + retryable: _, // TODO: https://github.com/FuelLabs/fuel-core/issues/2448 non_retryable, } = self .storage_as_ref::() From 53bbc7b378f52a45aaebdf2b4b560cbda51af0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 14:58:49 +0100 Subject: [PATCH 109/111] Revert the stray change in `balance.rs` --- crates/fuel-core/src/query/balance.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index dcc589118a3..706fcf02569 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -103,9 +103,7 @@ impl ReadView { let amount: &mut TotalBalanceAmount = amounts_per_asset .entry(*coin.asset_id(base_asset_id)) .or_default(); - let new_amount = - amount.saturating_add(coin.amount() as TotalBalanceAmount); - *amount = new_amount; + *amount = amount.saturating_add(coin.amount() as TotalBalanceAmount); Ok(amounts_per_asset) }, ) From cca18919cca2f828a6cc28785b2e074714a40f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Fri, 22 Nov 2024 16:56:01 +0100 Subject: [PATCH 110/111] Add basic storage tests for `CoinBalances` and `MessageBalances` --- .../src/graphql_api/storage/balances.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/fuel-core/src/graphql_api/storage/balances.rs b/crates/fuel-core/src/graphql_api/storage/balances.rs index 2d32af9afc0..a6402cd260e 100644 --- a/crates/fuel-core/src/graphql_api/storage/balances.rs +++ b/crates/fuel-core/src/graphql_api/storage/balances.rs @@ -81,3 +81,20 @@ impl TableWithBlueprint for MessageBalances { Self::Column::MessageBalances } } + +#[cfg(test)] +mod test { + use super::*; + + fuel_core_storage::basic_storage_tests!( + CoinBalances, + ::Key::default(), + ::Value::default() + ); + + fuel_core_storage::basic_storage_tests!( + MessageBalances, + ::Key::default(), + ::Value::default() + ); +} From be9b742936ae5fabcf152e6f036561e9a13beb88 Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Fri, 29 Nov 2024 14:44:40 +0100 Subject: [PATCH 111/111] Small suggestions and simplification to the balances indexation PR (#2465) Suggestions and simplifications for the https://github.com/FuelLabs/fuel-core/pull/2383. --- CHANGELOG.md | 2 - Cargo.lock | 2 +- crates/fuel-core/src/combined_database.rs | 45 +------ crates/fuel-core/src/database.rs | 13 +- .../src/database/database_description.rs | 19 +++ crates/fuel-core/src/database/metadata.rs | 18 +-- crates/fuel-core/src/graphql_api.rs | 5 +- .../fuel-core/src/graphql_api/indexation.rs | 93 +++++++------- crates/fuel-core/src/graphql_api/storage.rs | 4 +- .../src/query/balance/asset_query.rs | 2 +- crates/fuel-core/src/service.rs | 1 - .../service/adapters/graphql_api/off_chain.rs | 113 ++---------------- .../src/service/genesis/importer/off_chain.rs | 30 ++++- crates/types/src/entities.rs | 3 +- crates/types/src/entities/relayer/message.rs | 13 +- 15 files changed, 144 insertions(+), 219 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40472747691..6575dc0eb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/). #### Breaking - [2389](https://github.com/FuelLabs/fuel-core/pull/2258): Updated the `messageProof` GraphQL schema to return a non-nullable `MessageProof`. - -#### Breaking - [2383](https://github.com/FuelLabs/fuel-core/pull/2383): Asset balance queries now return U128 instead of U64. - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Transaction graphql endpoints use `TransactionType` instead of `fuel_tx::Transaction`. - [2446](https://github.com/FuelLabs/fuel-core/pull/2446): Use graphiql instead of graphql-playground due to known vulnerability and stale development. diff --git a/Cargo.lock b/Cargo.lock index 1ed9dd67683..07022867482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6958,7 +6958,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index c19b912c852..c0b6d291af1 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -7,11 +7,7 @@ use crate::{ off_chain::OffChain, on_chain::OnChain, relayer::Relayer, - DatabaseDescription, - DatabaseMetadata, - IndexationKind, }, - metadata::MetadataTable, Database, GenesisDatabase, Result as DatabaseResult, @@ -32,12 +28,7 @@ use fuel_core_storage::tables::{ ContractsState, Messages, }; -use fuel_core_storage::{ - transactional::ReadTransaction, - Result as StorageResult, - StorageAsMut, -}; -use fuel_core_txpool::ports::AtomicView; +use fuel_core_storage::Result as StorageResult; use fuel_core_types::fuel_types::BlockHeight; use std::path::PathBuf; @@ -178,40 +169,6 @@ impl CombinedDatabase { Ok(()) } - pub fn initialize(&self) -> StorageResult<()> { - self.initialize_indexation()?; - Ok(()) - } - - fn initialize_indexation(&self) -> StorageResult<()> { - // When genesis is missing write to the database that balances cache should be used. - let on_chain_view = self.on_chain().latest_view()?; - if on_chain_view.get_genesis().is_err() { - let all_indexations = IndexationKind::all().collect(); - tracing::info!( - "No genesis, initializing metadata with all supported indexations: {:?}", - all_indexations - ); - let off_chain_view = self.off_chain().latest_view()?; - let mut database_tx = off_chain_view.read_transaction(); - database_tx - .storage_as_mut::>() - .insert( - &(), - &DatabaseMetadata::V2 { - version: ::version(), - height: Default::default(), - indexation_availability: all_indexations, - }, - )?; - self.off_chain() - .data - .commit_changes(None, database_tx.into_changes())?; - }; - - Ok(()) - } - pub fn on_chain(&self) -> &Database { &self.on_chain } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 247b0bf2863..41d0451f221 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -24,7 +24,6 @@ use crate::{ KeyValueView, }, }; -use database_description::IndexationKind; use fuel_core_chain_config::TableEntry; use fuel_core_gas_price_service::common::fuel_core_storage_adapter::storage::GasPriceMetadata; use fuel_core_services::SharedMutex; @@ -68,7 +67,10 @@ pub use fuel_core_database::Error; pub type Result = core::result::Result; // TODO: Extract `Database` and all belongs into `fuel-core-database`. -use crate::database::database_description::gas_price::GasPriceDatabase; +use crate::database::database_description::{ + gas_price::GasPriceDatabase, + indexation_availability, +}; #[cfg(feature = "rocksdb")] use crate::state::{ historical_rocksdb::{ @@ -536,7 +538,7 @@ where None => DatabaseMetadata::V2 { version: Description::version(), height: new_height, - indexation_availability: IndexationKind::all().collect(), + indexation_availability: indexation_availability::(None), }, }; updated_metadata @@ -1115,12 +1117,12 @@ mod tests { } mod metadata { + use crate::database::database_description::IndexationKind; + use fuel_core_storage::kv_store::StorageColumn; use std::{ borrow::Cow, collections::HashSet, }; - - use fuel_core_storage::kv_store::StorageColumn; use strum::EnumCount; use super::{ @@ -1128,7 +1130,6 @@ mod tests { update_metadata, DatabaseHeight, DatabaseMetadata, - IndexationKind, }; #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index 9fb1fc73d5b..efc0f48b5bf 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -132,3 +132,22 @@ impl DatabaseMetadata { } } } + +/// Gets the indexation availability from the metadata. +pub fn indexation_availability( + metadata: Option>, +) -> HashSet +where + D: DatabaseDescription, +{ + match metadata { + Some(DatabaseMetadata::V1 { .. }) => HashSet::new(), + Some(DatabaseMetadata::V2 { + indexation_availability, + .. + }) => indexation_availability.clone(), + // If the metadata doesn't exist, it is a new database, + // and we should set all indexation kinds to available. + None => IndexationKind::all().collect(), + } +} diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 900a484a16c..bbc9a0fb353 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,3 +1,7 @@ +use super::database_description::{ + indexation_availability, + IndexationKind, +}; use crate::database::{ database_description::{ DatabaseDescription, @@ -17,8 +21,6 @@ use fuel_core_storage::{ StorageInspect, }; -use super::database_description::IndexationKind; - /// The table that stores all metadata about the database. pub struct MetadataTable(core::marker::PhantomData); @@ -78,10 +80,12 @@ where } pub fn indexation_available(&self, kind: IndexationKind) -> StorageResult { - let Some(metadata) = self.storage::>().get(&())? - else { - return Ok(false) - }; - Ok(metadata.indexation_available(kind)) + let metadata = self + .storage::>() + .get(&())? + .map(|metadata| metadata.into_owned()); + + let indexation_availability = indexation_availability::(metadata); + Ok(indexation_availability.contains(&kind)) } } diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 7472dfd2591..63a6efcf0de 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -80,7 +80,10 @@ impl Default for Costs { } pub const DEFAULT_QUERY_COSTS: Costs = Costs { - balance_query: 40001, /* Cost will depend on whether balances index is available or not, but let's keep the default high to be on the safe side */ + // TODO: The cost of the `balance` and `balances` query should depend on the + // `OffChainDatabase::balances_enabled` value. If additional indexation is enabled, + // the cost should be cheaper. + balance_query: 40001, coins_to_spend: 40001, get_peers: 40001, estimate_predicates: 40001, diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index fde469e9100..1b490c2f13d 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -65,12 +65,12 @@ where { let key = message.recipient(); let storage = block_st_transaction.storage::(); - let current_balance = storage.get(key)?.unwrap_or_default(); + let current_balance = storage.get(key)?.unwrap_or_default().into_owned(); let MessageBalance { mut retryable, mut non_retryable, - } = *current_balance; - if message.has_retryable_amount() { + } = current_balance; + if message.is_retryable_message() { retryable = retryable.saturating_add(message.amount() as u128); } else { non_retryable = non_retryable.saturating_add(message.amount() as u128); @@ -80,8 +80,10 @@ where non_retryable, }; - let storage = block_st_transaction.storage::(); - Ok(storage.insert(key, &new_balance)?) + block_st_transaction + .storage::() + .insert(key, &new_balance) + .map_err(Into::into) } fn decrease_message_balance( @@ -96,36 +98,37 @@ where let MessageBalance { retryable, non_retryable, - } = *storage.get(key)?.unwrap_or_default(); - let current_balance = if message.has_retryable_amount() { + } = storage.get(key)?.unwrap_or_default().into_owned(); + let current_balance = if message.is_retryable_message() { retryable } else { non_retryable }; - current_balance + let new_amount = current_balance .checked_sub(message.amount() as u128) .ok_or_else(|| IndexationError::MessageBalanceWouldUnderflow { owner: *message.recipient(), current_amount: current_balance, requested_deduction: message.amount() as u128, - retryable: message.has_retryable_amount(), - }) - .and_then(|new_amount| { - let storage = block_st_transaction.storage::(); - let new_balance = if message.has_retryable_amount() { - MessageBalance { - retryable: new_amount, - non_retryable, - } - } else { - MessageBalance { - retryable, - non_retryable: new_amount, - } - }; - storage.insert(key, &new_balance).map_err(Into::into) - }) + retryable: message.is_retryable_message(), + })?; + + let new_balance = if message.is_retryable_message() { + MessageBalance { + retryable: new_amount, + non_retryable, + } + } else { + MessageBalance { + retryable, + non_retryable: new_amount, + } + }; + block_st_transaction + .storage::() + .insert(key, &new_balance) + .map_err(Into::into) } fn increase_coin_balance( @@ -137,11 +140,13 @@ where { let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); let storage = block_st_transaction.storage::(); - let current_amount = *storage.get(&key)?.unwrap_or_default(); + let current_amount = storage.get(&key)?.unwrap_or_default().into_owned(); let new_amount = current_amount.saturating_add(coin.amount as u128); - let storage = block_st_transaction.storage::(); - Ok(storage.insert(&key, &new_amount)?) + block_st_transaction + .storage::() + .insert(&key, &new_amount) + .map_err(Into::into) } fn decrease_coin_balance( @@ -153,22 +158,22 @@ where { let key = CoinBalancesKey::new(&coin.owner, &coin.asset_id); let storage = block_st_transaction.storage::(); - let current_amount = *storage.get(&key)?.unwrap_or_default(); - - current_amount - .checked_sub(coin.amount as u128) - .ok_or_else(|| IndexationError::CoinBalanceWouldUnderflow { - owner: coin.owner, - asset_id: coin.asset_id, - current_amount, - requested_deduction: coin.amount as u128, - }) - .and_then(|new_amount| { - block_st_transaction - .storage::() - .insert(&key, &new_amount) - .map_err(Into::into) - }) + let current_amount = storage.get(&key)?.unwrap_or_default().into_owned(); + + let new_amount = + current_amount + .checked_sub(coin.amount as u128) + .ok_or_else(|| IndexationError::CoinBalanceWouldUnderflow { + owner: coin.owner, + asset_id: coin.asset_id, + current_amount, + requested_deduction: coin.amount as u128, + })?; + + block_st_transaction + .storage::() + .insert(&key, &new_amount) + .map_err(Into::into) } pub(crate) fn process_balances_update( diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs index de1db10a550..8ebe615b30a 100644 --- a/crates/fuel-core/src/graphql_api/storage.rs +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -114,9 +114,9 @@ pub enum Column { DaCompressionTemporalRegistryScriptCode = 21, /// See [`DaCompressionTemporalRegistryPredicateCode`](da_compression::DaCompressionTemporalRegistryPredicateCode) DaCompressionTemporalRegistryPredicateCode = 22, - /// Coin balances per user and asset. + /// Coin balances per account and asset. CoinBalances = 23, - /// Message balances per user. + /// Message balances per account. MessageBalances = 24, } diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index a7ddd4a5f05..6ea62390fd1 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -175,7 +175,7 @@ impl<'a> AssetsQuery<'a> { .try_flatten() .filter(|result| { if let Ok(message) = result { - !message.has_retryable_amount() + message.is_non_retryable_message() } else { true } diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index e5f81245141..1f751f1de5e 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -126,7 +126,6 @@ impl FuelService { // initialize state tracing::info!("Initializing database"); database.check_version()?; - database.initialize()?; Self::make_database_compatible_with_config( &mut database, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 7acd5d47f74..5f08234f38a 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use crate::{ database::{ database_description::{ @@ -34,9 +32,7 @@ use crate::{ OldFuelBlocks, OldTransactions, }, - Column, }, - state::IterableKeyValueView, }; use fuel_core_storage::{ blueprint::BlueprintInspect, @@ -57,7 +53,6 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, StorageAsRef, - StorageRef, }; use fuel_core_types::{ blockchain::{ @@ -83,6 +78,7 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; +use std::iter; impl OffChainDatabase for OffChainIterableKeyValueView { fn block_height(&self, id: &BlockId) -> StorageResult { @@ -244,12 +240,17 @@ impl OffChainDatabase for OffChainIterableKeyValueView { direction: IterDirection, ) -> BoxedIter<'_, StorageResult<(AssetId, TotalBalanceAmount)>> { let base_asset_id = *base_asset_id; - let base_asset_balance = base_asset_balance( - self.storage_as_ref::(), - self.storage_as_ref::(), - owner, - &base_asset_id, - ); + let base_balance = self.balance(owner, &base_asset_id, &base_asset_id); + let base_asset_balance = match base_balance { + Ok(base_asset_balance) => { + if base_asset_balance != 0 { + iter::once(Ok((base_asset_id, base_asset_balance))).into_boxed() + } else { + iter::empty().into_boxed() + } + } + Err(err) => iter::once(Err(err)).into_boxed(), + }; let non_base_asset_balance = self .iter_all_filtered_keys::(Some(owner), None, Some(direction)) @@ -283,96 +284,6 @@ impl OffChainDatabase for OffChainIterableKeyValueView { } } -struct AssetBalanceWithRetrievalErrors<'a> { - balance: Option, - errors: BoxedIter<'a, Result<(AssetId, u128), StorageError>>, -} - -impl<'a> AssetBalanceWithRetrievalErrors<'a> { - fn new( - balance: Option, - errors: BoxedIter<'a, Result<(AssetId, u128), StorageError>>, - ) -> Self { - Self { balance, errors } - } -} - -fn non_retryable_message_balance<'a>( - storage: StorageRef<'a, IterableKeyValueView, MessageBalances>, - owner: &Address, -) -> AssetBalanceWithRetrievalErrors<'a> { - storage.get(owner).map_or_else( - |err| { - AssetBalanceWithRetrievalErrors::new( - None, - std::iter::once(Err(err)).into_boxed(), - ) - }, - |value| { - AssetBalanceWithRetrievalErrors::new( - value.map(|value| value.non_retryable), - std::iter::empty().into_boxed(), - ) - }, - ) -} - -fn base_asset_coin_balance<'a, 'b>( - storage: StorageRef<'a, IterableKeyValueView, CoinBalances>, - owner: &'b Address, - base_asset_id: &'b AssetId, -) -> AssetBalanceWithRetrievalErrors<'a> { - storage - .get(&CoinBalancesKey::new(owner, base_asset_id)) - .map_or_else( - |err| { - AssetBalanceWithRetrievalErrors::new( - None, - std::iter::once(Err(err)).into_boxed(), - ) - }, - |value| { - AssetBalanceWithRetrievalErrors::new( - value.map(Cow::into_owned), - std::iter::empty().into_boxed(), - ) - }, - ) -} - -fn base_asset_balance<'a, 'b>( - messages_storage: StorageRef<'a, IterableKeyValueView, MessageBalances>, - coins_storage: StorageRef<'a, IterableKeyValueView, CoinBalances>, - owner: &'b Address, - base_asset_id: &'b AssetId, -) -> BoxedIter<'a, Result<(AssetId, TotalBalanceAmount), StorageError>> { - let AssetBalanceWithRetrievalErrors { - balance: messages_balance, - errors: message_errors, - } = non_retryable_message_balance(messages_storage, owner); - - let AssetBalanceWithRetrievalErrors { - balance: base_coin_balance, - errors: coin_errors, - } = base_asset_coin_balance(coins_storage, owner, base_asset_id); - - let base_asset_id = *base_asset_id; - let balance = match (messages_balance, base_coin_balance) { - (None, None) => None, - (None, Some(balance)) | (Some(balance), None) => Some(balance), - (Some(msg_balance), Some(coin_balance)) => { - Some(msg_balance.saturating_add(coin_balance)) - } - } - .into_iter() - .map(move |balance| Ok((base_asset_id, balance))); - - message_errors - .chain(coin_errors) - .chain(balance) - .into_boxed() -} - impl worker::OffChainDatabase for Database { type Transaction<'a> = StorageTransaction<&'a mut Self> where Self: 'a; diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 745b6b92a96..8726c73a235 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -43,8 +43,22 @@ use super::{ Handler, }; -// We always want to enable balances indexation if we're starting at genesis. -const BALANCES_INDEXATION_ENABLED: bool = true; +fn balances_indexation_enabled() -> bool { + use std::sync::OnceLock; + + static BALANCES_INDEXATION_ENABLED: OnceLock = OnceLock::new(); + + *BALANCES_INDEXATION_ENABLED.get_or_init(|| { + // During re-genesis process the metadata is always doesn't exist. + let metadata = None; + let indexation_availability = + crate::database::database_description::indexation_availability::( + metadata, + ); + indexation_availability + .contains(&crate::database::database_description::IndexationKind::Balances) + }) +} impl ImportTable for Handler { type TableInSnapshot = TransactionStatuses; @@ -113,7 +127,11 @@ impl ImportTable for Handler { let events = group .into_iter() .map(|TableEntry { value, .. }| Cow::Owned(Event::MessageImported(value))); - worker_service::process_executor_events(events, tx, BALANCES_INDEXATION_ENABLED)?; + worker_service::process_executor_events( + events, + tx, + balances_indexation_enabled(), + )?; Ok(()) } } @@ -131,7 +149,11 @@ impl ImportTable for Handler { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) }); - worker_service::process_executor_events(events, tx, BALANCES_INDEXATION_ENABLED)?; + worker_service::process_executor_events( + events, + tx, + balances_indexation_enabled(), + )?; Ok(()) } } diff --git a/crates/types/src/entities.rs b/crates/types/src/entities.rs index 5dbf816fba5..19c5ec0bf14 100644 --- a/crates/types/src/entities.rs +++ b/crates/types/src/entities.rs @@ -22,10 +22,9 @@ impl TryFrom for MessageCoin { let recipient = *message.recipient(); let nonce = *message.nonce(); let amount = message.amount(); - let data = message.data(); let da_height = message.da_height(); - if !data.is_empty() { + if message.is_retryable_message() { return Err(anyhow::anyhow!( "The data is not empty, impossible to convert into the `MessageCoin`" )) diff --git a/crates/types/src/entities/relayer/message.rs b/crates/types/src/entities/relayer/message.rs index 9ed46ffbbfc..5be4cbda490 100644 --- a/crates/types/src/entities/relayer/message.rs +++ b/crates/types/src/entities/relayer/message.rs @@ -135,9 +135,16 @@ impl Message { } } - /// Returns true if the message has retryable amount. - pub fn has_retryable_amount(&self) -> bool { - !self.data().is_empty() + /// Returns `true` if the message is retryable. + pub fn is_retryable_message(&self) -> bool { + let is_data_empty = self.data().is_empty(); + !is_data_empty + } + + /// Returns `true` if the message is non retryable. + pub fn is_non_retryable_message(&self) -> bool { + let is_data_empty = self.data().is_empty(); + is_data_empty } /// Set the message data