From 737c27392cfb35dea340ca72f6466918e9b0443f Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 19 Dec 2024 08:52:17 +0000 Subject: [PATCH 01/18] impl #BE-1771: Library provider updated --- client/src/contract/error.rs | 49 +++++++-- client/src/contract/factory/builder.rs | 24 ++-- client/src/contract/factory/library_loader.rs | 90 ++++++++------- .../src/contract/factory/library_provider.rs | 75 ++++++++++--- client/src/contract/jetton/master_contract.rs | 2 +- client/src/contract/jetton/wallet_contract.rs | 2 +- .../src/contract/nft/collection_contract.rs | 4 +- client/src/contract/nft/item_contract.rs | 2 +- client/src/contract/state.rs | 22 +++- client/src/tl/function.rs | 16 +++ client/src/types/ton_method_id.rs | 18 +++ client/src/types/tvm_stack_entry.rs | 5 + client/src/types/tvm_success.rs | 2 +- client/tests/contract_emulator_test.rs | 91 ++++++++++++++- client/tests/library_test.rs | 80 ++++++++++++++ core/src/cell.rs | 104 +++++++++--------- core/src/types.rs | 7 ++ core/src/types/address.rs | 2 +- 18 files changed, 449 insertions(+), 146 deletions(-) create mode 100644 client/tests/library_test.rs diff --git a/client/src/contract/error.rs b/client/src/contract/error.rs index 720d1d48..88bb47b5 100644 --- a/client/src/contract/error.rs +++ b/client/src/contract/error.rs @@ -1,7 +1,9 @@ +use std::borrow::Cow; use std::sync::Arc; use thiserror::Error; use tonlib_core::cell::TonCellError; +use tonlib_core::types::TonHashParseError; use tonlib_core::TonAddress; use crate::client::TonClientError; @@ -96,13 +98,16 @@ pub enum TonContractError { #[error("{0}")] CacheError(#[from] Arc), + + #[error("{0}")] + TonLibraryError(#[from] TonLibraryError), } pub trait MapStackError { #[allow(clippy::result_large_err)] fn map_stack_error( self, - method: &'static str, + method: impl Into>, address: &TonAddress, ) -> Result; } @@ -111,7 +116,7 @@ pub trait MapCellError { #[allow(clippy::result_large_err)] fn map_cell_error( self, - method: &'static str, + method: impl Into>, address: &TonAddress, ) -> Result; } @@ -119,25 +124,27 @@ pub trait MapCellError { impl MapStackError for Result { fn map_stack_error( self, - method: &'static str, + method: impl Into>, address: &TonAddress, ) -> Result { - self.map_err(|e| TonContractError::MethodResultStackError { - method: method.into(), - address: address.clone(), - error: e, - }) + self.map_err( + |e: TvmStackError| TonContractError::MethodResultStackError { + method: TonMethodId::Name(method.into()), + address: address.clone(), + error: e, + }, + ) } } impl MapStackError for Result { fn map_stack_error( self, - method: &'static str, + method: impl Into>, address: &TonAddress, ) -> Result { self.map_err(|e| TonContractError::TvmStackParseError { - method: method.into(), + method: TonMethodId::Name(method.into()), address: address.clone(), error: e, }) @@ -147,13 +154,31 @@ impl MapStackError for Result { impl MapCellError for Result { fn map_cell_error( self, - method: &'static str, + method: impl Into>, address: &TonAddress, ) -> Result { self.map_err(|e| TonContractError::MethodResultStackError { - method: method.into(), + method: TonMethodId::Name(method.into()), address: address.clone(), error: e.into(), }) } } + +#[derive(Error, Debug)] +pub enum TonLibraryError { + #[error("{0}")] + TonClientError(#[from] TonClientError), + + #[error("{0}")] + TonCellError(#[from] TonCellError), + + #[error("{0}")] + TonHashParseError(#[from] TonHashParseError), + + #[error("Library not found for {0}")] + LibraryNotFound(String), + + #[error("Expected exactly one library, but got multiple")] + MultipleLibrariesReturned, +} diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index f87123ba..136bf654 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use super::{DefaultLibraryLoader, LibraryProvider}; +use super::{DefaultLibraryLoader, LibraryLoader, LibraryProvider}; use crate::client::TonClient; use crate::contract::{TonContractError, TonContractFactory}; @@ -27,7 +27,7 @@ impl TonContractFactoryBuilder { pub(crate) fn new(client: &TonClient) -> Self { let loader = DefaultLibraryLoader::new(client); - let library_provider = LibraryProvider::new(Arc::new(loader)); + let library_provider = LibraryProvider::new(loader); TonContractFactoryBuilder { client: client.clone(), with_cache: false, @@ -40,13 +40,13 @@ impl TonContractFactoryBuilder { } } - pub fn with_account_state_cache( - &mut self, + pub fn with_cache( + mut self, txid_cache_capacity: u64, txid_cache_time_to_live: Duration, account_state_cache_capacity: u64, account_state_cache_time_to_live: Duration, - ) -> &mut Self { + ) -> Self { self.with_cache = true; self.txid_cache_capacity = txid_cache_capacity; self.txid_cache_time_to_live = txid_cache_time_to_live; @@ -55,7 +55,7 @@ impl TonContractFactoryBuilder { self } - pub fn with_default_cache(&mut self) -> &mut Self { + pub fn with_default_cache(mut self) -> Self { self.with_cache = true; self.account_state_cache_capacity = Self::DEFAULT_ACCOUNT_STATE_CACHE_CAPACITY; self.account_state_cache_time_to_live = Self::DEFAULT_ACCOUNT_STATE_CACHE_TTL; @@ -64,7 +64,7 @@ impl TonContractFactoryBuilder { self } - pub fn presync_blocks(&mut self, presync_blocks: i32) -> &mut Self { + pub fn with_presync_blocks(mut self, presync_blocks: i32) -> Self { self.presync_blocks = presync_blocks; self } @@ -82,17 +82,13 @@ impl TonContractFactoryBuilder { ) .await } -} - -impl TonContractFactoryBuilder { - pub fn with_default_library_provider(&mut self) -> &mut Self { - let loader = DefaultLibraryLoader::new(&self.client); - let library_provider = LibraryProvider::new(Arc::new(loader)); + pub fn with_library_loader(mut self, library_loader: &Arc) -> Self { + let library_provider = LibraryProvider::new(library_loader.clone()); self.library_provider = library_provider; self } - pub fn with_library_provider(&mut self, library_provider: &LibraryProvider) -> &mut Self { + pub fn with_library_provider(mut self, library_provider: &LibraryProvider) -> Self { self.library_provider = library_provider.clone(); self } diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index f51d6526..0c5b7bb1 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -1,14 +1,14 @@ use std::sync::Arc; use async_trait::async_trait; -use base64::engine::general_purpose::STANDARD; -use base64::Engine; -use tonlib_core::TonAddress; +use tonlib_core::cell::{ArcCell, BagOfCells}; +use tonlib_core::TonHash; use crate::client::{TonClient, TonClientInterface}; -use crate::contract::TonContractError; -use crate::tl::{SmcLibraryQueryExt, TonLibraryId}; +use crate::contract::TonLibraryError; +use crate::tl::TonLibraryId; +#[derive(Debug)] pub struct ContractLibraryDict { pub dict_boc: Vec, pub keys: Vec, @@ -16,11 +16,9 @@ pub struct ContractLibraryDict { #[async_trait] pub trait LibraryLoader: Send + Sync { - async fn load_contract_libraries( - &self, - address: &TonAddress, - code: &[u8], - ) -> Result, TonContractError>; + async fn get_library(&self, hash: &TonHash) -> Result, TonLibraryError>; + + async fn get_libraries(&self, hashes: &[TonHash]) -> Result, TonLibraryError>; } pub struct DefaultLibraryLoader { @@ -28,42 +26,58 @@ pub struct DefaultLibraryLoader { } impl DefaultLibraryLoader { - pub fn new(client: &TonClient) -> Self { - DefaultLibraryLoader { + pub fn new(client: &TonClient) -> Arc { + Arc::new(DefaultLibraryLoader { client: client.clone(), - } + }) } } #[async_trait] impl LibraryLoader for DefaultLibraryLoader { - async fn load_contract_libraries( - &self, - address: &TonAddress, - code: &[u8], - ) -> Result, TonContractError> { - const DEFAULT_MAX_LIBS: i32 = 255; - let library_query = SmcLibraryQueryExt::ScanBoc { - boc: code.to_vec(), - max_libs: DEFAULT_MAX_LIBS, - }; - let library_result = self.client.smc_get_libraries_ext(&[library_query]).await?; - if !library_result.libs_not_found.is_empty() { - let missing_libs = library_result - .libs_not_found - .iter() - .map(|l| STANDARD.encode(&l.id)) - .collect(); - return Err(TonContractError::LibraryNotFound { - address: address.clone(), - missing_library: missing_libs, - }); + async fn get_library(&self, hash: &TonHash) -> Result, TonLibraryError> { + let library_result = self.get_libraries(&[*hash]).await?; + match library_result.len() { + 0 => { + log::warn!("Library not found for {:?}", hash); + Ok(None) + } + 1 => Ok(Some(library_result[0].clone())), + _ => Err(TonLibraryError::MultipleLibrariesReturned), + } + } + + async fn get_libraries(&self, hashes: &[TonHash]) -> Result, TonLibraryError> { + let mut results = Vec::new(); + + // If hashes exceed MAX_LIBS_REQUESTED, split them into chunks + for chunk in hashes.chunks(Self::MAX_LIBS_REQUESTED) { + let mut partial_result = self.get_libraries_impl(chunk).await?; + results.append(&mut partial_result); } + Ok(results) + } +} + +impl DefaultLibraryLoader { + const MAX_LIBS_REQUESTED: usize = 255; + + async fn get_libraries_impl( + &self, + hashes: &[TonHash], + ) -> Result, TonLibraryError> { + let library_list: Vec<_> = hashes + .iter() + .map(|hash| TonLibraryId::from(*hash)) + .collect(); + let library_result = self.client.smc_get_libraries(&library_list).await?; - let dict_boc = library_result.dict_boc; - let keys = library_result.libs_ok; + let libraries: Vec = library_result + .result + .into_iter() + .map(|lib| BagOfCells::parse(&lib.data)?.into_single_root()) + .collect::>()?; - let contract_libraies = ContractLibraryDict { dict_boc, keys }; - Ok(Arc::new(contract_libraies)) + Ok(libraries) } } diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 18414673..90688c24 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -1,10 +1,12 @@ +use std::collections::{HashSet, VecDeque}; use std::sync::Arc; -use tonlib_core::TonAddress; +use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; +use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; +use tonlib_core::TonHash; use super::{ContractLibraryDict, LibraryLoader}; -use crate::contract::TonContractError; -use crate::tl::RawFullAccountState; +use crate::contract::TonLibraryError; #[derive(Clone)] pub struct LibraryProvider { @@ -12,25 +14,62 @@ pub struct LibraryProvider { } impl LibraryProvider { - pub fn new(loader: Arc) -> LibraryProvider { - LibraryProvider { loader } + pub fn new(loader: Arc) -> Self { + Self { loader } } - pub async fn get_contract_libraries( - &self, - address: &TonAddress, - account_state: &Arc, - ) -> Result, TonContractError> { - self.get_libraries_by_contract_code(address, &account_state.code) - .await + pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> + where + I: IntoIterator, + { + let mut arc_cell_buffer = VecDeque::new(); + + arc_cell_buffer.extend(cells); + + let mut lib_cells = HashSet::new(); + let mut visited_cells = HashSet::new(); + + while let Some(cell) = arc_cell_buffer.pop_front() { + if !visited_cells.insert(cell) { + continue; + } + let refs = cell.references(); + arc_cell_buffer.extend(refs.iter()); + if cell.is_library() { + lib_cells.insert(cell.clone()); + } + } + + let lib_hashes: Vec = lib_cells + .iter() + .map(|i| i.data()[1..].try_into()) + .collect::>()?; + + Ok(lib_hashes) } - pub async fn get_libraries_by_contract_code( + pub async fn get_libs_dict<'a, I>( &self, - address: &TonAddress, - code: &[u8], - ) -> Result, TonContractError> { - // TODO cache - self.loader.load_contract_libraries(address, code).await + cells: I, + ) -> Result + where + I: IntoIterator, + { + let refs = LibraryProvider::extract_library_hashes(cells)?; + + let libs = self.loader.get_libraries(refs.as_slice()).await?; + + let lib_hashmap = libs.into_iter().map(|l| (l.cell_hash(), l)).collect(); + + let lib_cell = CellBuilder::new() + .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? + .build()?; + + let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; + + let keys = refs.iter().map(|r| (*r).into()).collect(); + + let dict = ContractLibraryDict { dict_boc, keys }; + Ok(dict) } } diff --git a/client/src/contract/jetton/master_contract.rs b/client/src/contract/jetton/master_contract.rs index 50029209..f973b48d 100644 --- a/client/src/contract/jetton/master_contract.rs +++ b/client/src/contract/jetton/master_contract.rs @@ -28,7 +28,7 @@ enum JettonMasterMethods { pub trait JettonMasterContract: TonContractInterface { async fn get_jetton_data(&self) -> Result { const JETTON_DATA_STACK_ELEMENTS: usize = 5; - let method = JettonMasterMethods::GetJettonData.into(); + let method: &str = JettonMasterMethods::GetJettonData.into(); let address = self.address().clone(); let res = self.run_get_method(method, Vec::new()).await?; diff --git a/client/src/contract/jetton/wallet_contract.rs b/client/src/contract/jetton/wallet_contract.rs index 5dce818f..745e85f1 100644 --- a/client/src/contract/jetton/wallet_contract.rs +++ b/client/src/contract/jetton/wallet_contract.rs @@ -24,7 +24,7 @@ enum JettonWalletMethods { pub trait JettonWalletContract: TonContractInterface { async fn get_wallet_data(&self) -> Result { const WALLET_DATA_STACK_ELEMENTS: usize = 4; - let method = JettonWalletMethods::GetWalletData.into(); + let method: &str = JettonWalletMethods::GetWalletData.into(); let address = self.address().clone(); let res = self.run_get_method(method, Vec::new()).await?; diff --git a/client/src/contract/nft/collection_contract.rs b/client/src/contract/nft/collection_contract.rs index 977759f7..09bab829 100644 --- a/client/src/contract/nft/collection_contract.rs +++ b/client/src/contract/nft/collection_contract.rs @@ -38,7 +38,7 @@ pub trait NftCollectionContract: TonContractInterface { /// Returns nft collection data. async fn get_collection_data(&self) -> Result { const NFT_COLLECTION_STACK_ELEMENTS: usize = 3; - let method = NftCollectionMethods::GetCollectionData.into(); + let method: &str = NftCollectionMethods::GetCollectionData.into(); let address = self.address().clone(); let stack = self.run_get_method(method, Vec::new()).await?.stack; @@ -67,7 +67,7 @@ pub trait NftCollectionContract: TonContractInterface { /// Gets the serial number of the NFT item of this collection and /// returns the address (TonAddress) of this NFT item smart contract. async fn get_nft_address_by_index(&self, index: i64) -> Result { - let method = NftCollectionMethods::GetNftAddressByIndex.into(); + let method: &str = NftCollectionMethods::GetNftAddressByIndex.into(); let input_stack = vec![TvmStackEntry::Int64(index)]; let stack = self.run_get_method(method, &input_stack).await?.stack; diff --git a/client/src/contract/nft/item_contract.rs b/client/src/contract/nft/item_contract.rs index eb23c751..5e3f031d 100644 --- a/client/src/contract/nft/item_contract.rs +++ b/client/src/contract/nft/item_contract.rs @@ -39,7 +39,7 @@ enum NftItemContractMethods { #[async_trait] pub trait NftItemContract: TonContractInterface { async fn get_nft_data(&self) -> Result { - let method = NftItemContractMethods::GetNftData.into(); + let method: &str = NftItemContractMethods::GetNftData.into(); const NFT_DATA_STACK_ELEMENTS: usize = 5; let address = self.address().clone(); diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index 101ab085..91c41dff 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -1,9 +1,10 @@ use std::sync::Arc; use async_trait::async_trait; -use tonlib_core::cell::Cell; +use tonlib_core::cell::{BagOfCells, Cell}; use tonlib_core::TonAddress; +use super::MapCellError; use crate::client::{TonClientError, TonClientInterface}; use crate::contract::{TonContractError, TonContractFactory, TonContractInterface}; use crate::emulator::c7_register::TvmEmulatorC7; @@ -81,7 +82,8 @@ impl TonContractState { M: Into + Send + Copy, S: AsRef<[TvmStackEntry]> + Send, { - let method_id = &method.into(); + let method_id: TonMethodId = method.into(); + let method_str = method_id.as_str(); let stack_ref = stack.as_ref(); let state = self.account_state.clone(); let c7 = TvmEmulatorC7::new( @@ -89,10 +91,18 @@ impl TonContractState { self.factory.get_config_cell_serial().await?.to_vec(), )?; + let code = BagOfCells::parse(&self.account_state.code) + .and_then(|mut boc| boc.into_single_root()) + .map_cell_error(method_str.clone(), &self.address)?; + + let data = BagOfCells::parse(&self.account_state.data) + .and_then(|mut boc| boc.into_single_root()) + .map_cell_error(method_str, &self.address)?; + let libs = self .factory .library_provider() - .get_contract_libraries(&self.address, &self.account_state) + .get_libs_dict(&[code, data]) .await?; let run_result = unsafe { @@ -102,7 +112,7 @@ impl TonContractState { // outlive spawned future. // But we're know it for sure since we're awaiting it. In normal async/await block // this would be checked by the compiler, but not when using `spawn_blocking` - let static_method_id: &'static TonMethodId = std::mem::transmute(method_id); + let static_method_id: TonMethodId = method_id.clone(); let static_stack: &'static [TvmStackEntry] = std::mem::transmute(stack_ref); #[allow(clippy::let_and_return)] tokio::task::spawn_blocking(move || { @@ -111,7 +121,7 @@ impl TonContractState { let mut emulator = TvmEmulator::new(code, data)? .with_c7(&c7)? .with_libraries(libs.dict_boc.as_slice())?; - let run_result = emulator.run_get_method(static_method_id, static_stack); + let run_result = emulator.run_get_method(&static_method_id, static_stack); run_result }) .await @@ -122,7 +132,7 @@ impl TonContractState { address: self.address().clone(), error: e, }); - Self::raise_exit_error(self.address(), method_id, run_result?) + Self::raise_exit_error(self.address(), &method_id, run_result?) } pub async fn emulate_internal_message( diff --git a/client/src/tl/function.rs b/client/src/tl/function.rs index e7543272..20c82e32 100644 --- a/client/src/tl/function.rs +++ b/client/src/tl/function.rs @@ -2,6 +2,8 @@ use base64::engine::general_purpose::STANDARD; use base64::Engine; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::IntoStaticStr; +use tonlib_core::types::TonHashParseError; +use tonlib_core::TonHash; use crate::tl::stack::TvmStackEntry; use crate::tl::types::{ @@ -42,6 +44,20 @@ impl<'de> Deserialize<'de> for TonLibraryId { } } +impl From for TonLibraryId { + fn from(value: TonHash) -> Self { + TonLibraryId { id: value.to_vec() } + } +} + +impl TryFrom for TonHash { + type Error = TonHashParseError; + + fn try_from(value: TonLibraryId) -> Result { + value.id.try_into() + } +} + #[derive(IntoStaticStr, Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] #[serde(tag = "@type", rename_all = "camelCase")] pub enum TonFunction { diff --git a/client/src/types/ton_method_id.rs b/client/src/types/ton_method_id.rs index 6c86bf76..4a0a0bdb 100644 --- a/client/src/types/ton_method_id.rs +++ b/client/src/types/ton_method_id.rs @@ -29,6 +29,24 @@ impl From<&'static str> for TonMethodId { } } +impl TonMethodId { + pub fn as_str(&self) -> Cow<'static, str> { + match self { + TonMethodId::Number(num) => Cow::Owned(num.to_string()), // Dynamically allocate for number + TonMethodId::Name(cow) => match cow { + Cow::Borrowed(s) => Cow::Borrowed(*s), // Safe only if already 'static + Cow::Owned(s) => Cow::Owned(s.clone()), // Clone the owned String + }, + } + } +} + +impl From> for TonMethodId { + fn from(value: Cow<'_, str>) -> Self { + TonMethodId::Name(Cow::Owned(value.into_owned())) + } +} + impl From for TonMethodId { fn from(value: String) -> Self { TonMethodId::Name(Cow::Owned(value)) diff --git a/client/src/types/tvm_stack_entry.rs b/client/src/types/tvm_stack_entry.rs index 9610c4de..c00c5064 100644 --- a/client/src/types/tvm_stack_entry.rs +++ b/client/src/types/tvm_stack_entry.rs @@ -88,6 +88,11 @@ impl TvmStackEntry { pub fn get_cell(&self) -> Result { match self { TvmStackEntry::Cell(cell) => Ok(cell.clone()), + TvmStackEntry::Slice(slice) => { + let cell = slice.into_cell()?; + Ok(cell.into()) + } + t => Err(StackParseError::InvalidEntryType { expected: "Cell".to_string(), found: t.clone(), diff --git a/client/src/types/tvm_success.rs b/client/src/types/tvm_success.rs index 580678a0..a47b062e 100644 --- a/client/src/types/tvm_success.rs +++ b/client/src/types/tvm_success.rs @@ -2,7 +2,7 @@ use tonlib_core::cell::ArcCell; use crate::types::TvmStackEntry; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct TvmSuccess { pub vm_log: Option, pub vm_exit_code: i32, diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 806a0948..7477424f 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -7,7 +7,9 @@ use tonlib_client::emulator::c7_register::TvmEmulatorC7; use tonlib_client::emulator::tvm_emulator::TvmEmulator; use tonlib_client::meta::MetaDataContent; use tonlib_client::tl::RawFullAccountState; -use tonlib_core::TonAddress; +use tonlib_client::types::TvmStackEntry; +use tonlib_core::cell::{CellBuilder, CellSlice}; +use tonlib_core::{TonAddress, TonTxId}; #[tokio::test] async fn test_emulator_get_jetton_data() -> anyhow::Result<()> { @@ -106,3 +108,90 @@ async fn test_emulator_get_wallet_address() { assert_eq!(r1.vm_exit_code, r2.vm_exit_code); assert_eq!(r1.vm_exit_code, r3.vm_exit_code); } + +#[tokio::test] +async fn test_emulate_ston_router_v2() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + let factory = TonContractFactory::builder(&client).build().await?; + + let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; + let tx_id = TonTxId::from_lt_hash( + 51600010000005, + "82218cf8373437ffeac1bf306f44d9638894c2d2b4b2bddf85ac2c571b56b2a7", + )?; + + let contract = factory.get_contract(&router_address); + let state = contract.get_state_by_transaction(&tx_id.into()).await?; + + let token1_addr = CellSlice::full_cell( + CellBuilder::new() + .store_address(&"EQC8JhkQsgAwRpe0lMsr6U11NXWjwgty22gxnRt_pSq4jDmb".parse()?)? + .build()?, + )?; + let token2_addr = CellSlice::full_cell( + CellBuilder::new() + .store_address(&"EQB1R5vBgbJBZNVkh55XID629E2Xq9MFib3nai9QSkZ2F7X4".parse()?)? + .build()?, + )?; + + let call_parameters_vec = [ + ("get_router_data", vec![]), + ("get_upgraded_pool_code", vec![]), + ("get_router_version", vec![]), + ( + "get_pool_address", + vec![ + TvmStackEntry::Slice(token1_addr), + TvmStackEntry::Slice(token2_addr), + ], + ), + ]; + for call_parameters in call_parameters_vec { + let method_id = call_parameters.0; + let result: tonlib_client::types::TvmSuccess = assert_ok!( + state + .emulate_get_method(method_id, call_parameters.1.as_slice()) + .await + ); + + let expected_result = state + .tonlib_run_get_method(method_id, call_parameters.1.as_slice()) + .await + .unwrap(); + + log::info!( + "Called router with method: {:?}, stack: {:?}", + call_parameters.0, + call_parameters.1 + ); + + log::info!("METHOD: {:?}", method_id); + log::info!("___________________Blockchain_result \n {:?} \n-------------------------------------------", expected_result); + log::info!("_____________________Emulated_result \n {:?} \n-------------------------------------------", result); + + assert_eq!(result.gas_used, expected_result.gas_used); + assert_eq!(result.missing_library, expected_result.missing_library); + assert_eq!(result.vm_exit_code, expected_result.vm_exit_code); + + //explicitly omitted check of vm_log, as it is not returned from blockchain + // assert_eq!(result.vm_log, expected_result.vm_log); + + for i in 0..expected_result.stack.len() { + let (expected, actual) = (expected_result.stack[i].clone(), result.stack[i].clone()); + + match (expected, actual) { + (TvmStackEntry::Cell(e), TvmStackEntry::Cell(a)) => assert_eq!(e, a), + + (TvmStackEntry::Slice(e), TvmStackEntry::Slice(a)) => { + assert_eq!(e.into_cell().unwrap(), a.into_cell().unwrap()) + } + + (TvmStackEntry::Int257(e), TvmStackEntry::Int64(a)) => assert_eq!(e, a.into()), + + (_, _) => panic!(), + } + } + } + Ok(()) +} diff --git a/client/tests/library_test.rs b/client/tests/library_test.rs new file mode 100644 index 00000000..6c208cd2 --- /dev/null +++ b/client/tests/library_test.rs @@ -0,0 +1,80 @@ +use std::str::FromStr; + +use tokio_test::assert_ok; +use tonlib_client::contract::{ + DefaultLibraryLoader, LibraryLoader, LibraryProvider, TonContractFactory, +}; +use tonlib_core::cell::BagOfCells; +use tonlib_core::{TonAddress, TonHash}; + +mod common; + +#[tokio::test] +async fn test_get_lib_by_hash() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + + let expected_lib_id = TonHash::from([ + 159, 49, 244, 244, 19, 163, 172, 203, 112, 108, 136, 150, 42, 198, 157, 89, 16, 59, 1, 58, + 10, 221, 207, 174, 237, 93, 215, 60, 24, 250, 152, 168, + ]); + log::info!("{:?}", expected_lib_id); + + let library_loader = DefaultLibraryLoader::new(&client); + let maybe_lib = library_loader.get_library(&expected_lib_id).await?; + assert!(maybe_lib.is_some()); + + let lib = maybe_lib.unwrap(); + let lib_hash = lib.cell_hash(); + + assert_eq!(expected_lib_id, lib_hash); + + Ok(()) +} + +#[tokio::test] +async fn test_get_libs_by_hash() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + + let expected_lib_id = TonHash::from([ + 159, 49, 244, 244, 19, 163, 172, 203, 112, 108, 136, 150, 42, 198, 157, 89, 16, 59, 1, 58, + 10, 221, 207, 174, 237, 93, 215, 60, 24, 250, 152, 168, + ]); + log::info!("{:?}", expected_lib_id); + + let library_loader = DefaultLibraryLoader::new(&client); + let maybe_lib = library_loader + .get_libraries(&[expected_lib_id, expected_lib_id]) + .await?; + + assert_eq!(maybe_lib.len(), 1); + assert_eq!(maybe_lib[0].cell_hash(), expected_lib_id); + Ok(()) +} + +#[tokio::test] +async fn test_get_lib_hashes_by_code() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + let factory = TonContractFactory::builder(&client).build().await?; + + let address = TonAddress::from_str("EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB")?; + + let state = factory.get_latest_account_state(&address).await?; + let code = BagOfCells::parse(&state.code)?.into_single_root()?; + + let hashes = assert_ok!(LibraryProvider::extract_library_hashes(&[code])); + + log::info!("{:?}", hashes); + + let expected_lib_id = TonHash::from([ + 159, 49, 244, 244, 19, 163, 172, 203, 112, 108, 136, 150, 42, 198, 157, 89, 16, 59, 1, 58, + 10, 221, 207, 174, 237, 93, 215, 60, 24, 250, 152, 168, + ]); + + assert_eq!(hashes.len(), 1); + assert_eq!(expected_lib_id, hashes[0]); + + Ok(()) +} diff --git a/core/src/cell.rs b/core/src/cell.rs index c09a61b9..ef5e34ad 100644 --- a/core/src/cell.rs +++ b/core/src/cell.rs @@ -1,8 +1,8 @@ -use std::fmt::{Debug, Formatter}; +use std::fmt::Debug; use std::hash::Hash; +use std::io; use std::ops::Deref; use std::sync::Arc; -use std::{fmt, io}; pub use bag_of_cells::*; use base64::Engine; @@ -155,6 +155,10 @@ impl Cell { self.cell_type != CellType::Ordinary } + pub fn is_library(&self) -> bool { + self.cell_type == CellType::Library + } + pub fn cell_hash_base64(&self) -> String { self.cell_hash().to_base64() } @@ -239,54 +243,8 @@ impl Cell { } impl Debug for Cell { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let t = match self.cell_type { - CellType::Ordinary | CellType::Library => 'x', - CellType::PrunedBranch | CellType::MerkleProof => 'p', - CellType::MerkleUpdate => 'u', - }; - - // Our completion tag ONLY shows that the last byte is incomplete - // It does not correspond to real completion tag defined in - // p1.0.2 of https://docs.ton.org/tvm.pdf for details - // Null termination of bit-string defined in that document is omitted for clarity - let completion_tag = if self.bit_len % 8 != 0 { "_" } else { "" }; - writeln!( - f, - "Cell {}{{ data: [{}{}]\n, bit_len: {}\n, references: [", - t, - self.data - .iter() - .map(|&byte| format!("{:02X}", byte)) - .collect::>() - .join(""), - completion_tag, - self.bit_len, - )?; - - for reference in &self.references { - writeln!( - f, - " {}\n", - format!("{:?}", reference).replace('\n', "\n ") - )?; - } - - write!( - f, - "]\n cell_type: {:?}\n level_mask: {:?}\n hashes {:?}\n depths {:?}\n }}", - self.cell_type, - self.level_mask, - self.hashes - .iter() - .map(|h| h - .iter() - .map(|&byte| format!("{:02X}", byte)) - .collect::>() - .join("")) - .collect::>(), - self.depths - ) + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write_cell_debug(f, self, 0) } } @@ -516,6 +474,52 @@ fn write_ref_hashes( Ok(()) } +fn write_cell_debug( + f: &mut std::fmt::Formatter<'_>, + cell: &Cell, + indent_level: usize, +) -> std::fmt::Result { + let indent = " ".repeat(indent_level); + // Generate the data display string + let mut data_display: String = cell.data.iter().fold(String::new(), |mut acc, &byte| { + acc.push_str(&format!("{:02X}", byte)); + acc + }); + + let data_display = if data_display.is_empty() { + "_" + } else { + // Our completion tag ONLY shows that the last byte is incomplete + // It does not correspond to real completion tag defined in + // p1.0.2 of https://docs.ton.org/tvm.pdf for details + // Null termination of bit-string defined in that document is omitted for clarity + if cell.bit_len % 8 != 0 { + data_display.push('_'); + } + &data_display + }; + + if cell.references.is_empty() { + // Compact format for cells without references + writeln!( + f, + "{}Cell x{{Type: {:?}, data: [{}], bit_len: {}}}", + indent, cell.cell_type, data_display, cell.bit_len + ) + } else { + // Full format for cells with references + writeln!( + f, + "{}Cell x{{Type: {:?}, data: [{}], bit_len: {}, references: [", + indent, cell.cell_type, data_display, cell.bit_len + )?; + for reference in &cell.references { + write_cell_debug(f, reference, indent_level + 1)?; + } + writeln!(f, "{}]}}", indent) + } +} + #[cfg(test)] mod test { use std::sync::Arc; diff --git a/core/src/types.rs b/core/src/types.rs index 4ed9e201..99e8927f 100644 --- a/core/src/types.rs +++ b/core/src/types.rs @@ -6,6 +6,7 @@ pub use address::*; use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; pub use error::*; +use num_bigint::BigUint; use serde::{Deserialize, Serialize}; pub use tx_id::*; @@ -82,6 +83,12 @@ impl From<&[u8; 32]> for TonHash { } } +impl From for BigUint { + fn from(value: TonHash) -> Self { + BigUint::from_bytes_be(value.as_slice()) + } +} + impl TryFrom<&[u8]> for TonHash { type Error = TonHashParseError; diff --git a/core/src/types/address.rs b/core/src/types/address.rs index 967bf233..c55e7ebd 100644 --- a/core/src/types/address.rs +++ b/core/src/types/address.rs @@ -514,7 +514,7 @@ mod tests { let address1 = TonAddress::from_str("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")?; let cmp_result = address0 < address1; - assert_eq!(true, cmp_result); + assert!(cmp_result); Ok(()) } } From d7f8d7a33b86e3ee69d2e1c866901e2395ba40c0 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 19 Dec 2024 10:04:15 +0000 Subject: [PATCH 02/18] Impl #BE-1843: Bencmark for contract emulator --- client/tests/contract_emulator_test.rs | 141 ++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 3 deletions(-) diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 7477424f..86126524 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -1,14 +1,16 @@ mod common; +use std::time::{Duration, Instant}; + use tokio_test::assert_ok; use tonlib_client::contract::{ - JettonData, JettonMasterContract, TonContractFactory, TonContractInterface, + JettonData, JettonMasterContract, TonContractError, TonContractFactory, TonContractInterface, }; use tonlib_client::emulator::c7_register::TvmEmulatorC7; use tonlib_client::emulator::tvm_emulator::TvmEmulator; use tonlib_client::meta::MetaDataContent; use tonlib_client::tl::RawFullAccountState; -use tonlib_client::types::TvmStackEntry; -use tonlib_core::cell::{CellBuilder, CellSlice}; +use tonlib_client::types::{TonMethodId, TvmStackEntry, TvmSuccess}; +use tonlib_core::cell::{BagOfCells, CellBuilder, CellSlice}; use tonlib_core::{TonAddress, TonTxId}; #[tokio::test] @@ -195,3 +197,136 @@ async fn test_emulate_ston_router_v2() -> anyhow::Result<()> { } Ok(()) } + +/// Benchmark for emulator +#[ignore] +#[tokio::test] +async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_archive_mainnet_client().await; + let factory = TonContractFactory::builder(&client).build().await?; + + let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; + let tx_id = TonTxId::from_lt_hash( + 51600010000005, + "82218cf8373437ffeac1bf306f44d9638894c2d2b4b2bddf85ac2c571b56b2a7", + )?; + + let contract = factory.get_contract(&router_address); + let state = contract.get_state_by_transaction(&tx_id.into()).await?; + + let token1_addr = CellSlice::full_cell( + CellBuilder::new() + .store_address(&"EQC8JhkQsgAwRpe0lMsr6U11NXWjwgty22gxnRt_pSq4jDmb".parse()?)? + .build()?, + )?; + let token2_addr = CellSlice::full_cell( + CellBuilder::new() + .store_address(&"EQB1R5vBgbJBZNVkh55XID629E2Xq9MFib3nai9QSkZ2F7X4".parse()?)? + .build()?, + )?; + + let stack = &vec![ + TvmStackEntry::Slice(token1_addr.clone()), + TvmStackEntry::Slice(token2_addr.clone()), + ]; + let stack_ref: &[TvmStackEntry] = stack.as_ref(); + + let code = state.get_account_state().code.clone(); + let data = state.get_account_state().data.clone(); + + let code_cell = BagOfCells::parse(&code)?.into_single_root()?; + let data_cell = BagOfCells::parse(&data)?.into_single_root()?; + + let c7 = TvmEmulatorC7::new( + router_address.clone(), + factory.get_config_cell_serial().await?.to_vec(), + )?; + let libs = factory + .library_provider() + .get_libs_dict(&[code_cell, data_cell]) + .await?; + + let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( + ( + Default::default(), + Default::default(), + Default::default(), + Default::default(), + ), + Default::default(), + ); + + const MAX_ITER: usize = 10; + for i in 0..MAX_ITER { + let run_result: ( + (TvmSuccess, Duration, Duration, Duration, Duration), + Duration, + ) = unsafe { + // Using unsafe to extend lifetime of references to method_id & stack. + // + // This is necessary because the compiler doesn't have a proof that these references + // outlive spawned future. + // But we're know it for sure since we're awaiting it. In normal async/await block + // this would be checked by the compiler, but not when using `spawn_blocking` + let static_method_id: TonMethodId = "get_pool_data".into(); + let static_stack: &'static [TvmStackEntry] = std::mem::transmute(stack_ref); + + let code = code.clone(); + let data = data.clone(); + let c7 = c7.clone(); + let libs = libs.dict_boc.clone(); + + #[allow(clippy::let_and_return)] + let ovetall_t = Instant::now(); + let res = tokio::task::spawn_blocking(move || { + let code = code.as_slice(); + let data = data.as_slice(); + + let t_creation = Instant::now(); + let emulator = TvmEmulator::new(code, data).unwrap(); + let creation_time = t_creation.elapsed(); + + let t_c7 = Instant::now(); + let e = emulator.with_c7(&c7).unwrap(); + let c7_time = t_c7.elapsed(); + + let t_lib = Instant::now(); + let mut e = e.with_libraries(libs.as_slice()).unwrap(); + let lib_time = t_lib.elapsed(); + + let running_time = Instant::now(); + let run_result = e.run_get_method(&static_method_id, static_stack); + let running_time = running_time.elapsed(); + ( + run_result.unwrap(), + creation_time, + c7_time, + lib_time, + running_time, + ) + }) + .await + .map_err(|e| TonContractError::InternalError(e.to_string()))?; + + let sum_t = ovetall_t.elapsed(); + (res, sum_t) + }; + + log::info!("{} of {}: creation_time: {:?}, c7_time: {:?}, lib_time: {:?}, running_time: {:?}, overall+tokio: {:?}", i, MAX_ITER,run_result.0.1,run_result.0.2,run_result.0.3,run_result.0.4,run_result.1); + + sums.0 .0 += run_result.0 .1; + sums.0 .1 += run_result.0 .2; + sums.0 .2 += run_result.0 .3; + sums.0 .3 += run_result.0 .4; + sums.1 += run_result.1 + } + + log::info!( + "_________________OVERALL over {}_________________", + MAX_ITER + ); + log::info!("creation_time: {:?}, c7_time: {:?}, lib_time: {:?}, running_time: {:?}, overall+tokio: {:?}", sums.0.0, sums.0.1,sums.0.2,sums.0.3,sums.1); + + Ok(()) +} From 5d9abe0d0c2b8c24a9a352fccbba1ad192ea600a Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 19 Dec 2024 13:41:50 +0000 Subject: [PATCH 03/18] NI: #BE-1771: fix vb comments --- client/src/contract/factory/builder.rs | 4 +- client/src/contract/factory/library_loader.rs | 40 ++++++++----------- .../src/contract/factory/library_provider.rs | 3 +- client/src/contract/state.rs | 2 +- client/tests/contract_emulator_test.rs | 2 +- client/tests/library_test.rs | 29 ++------------ 6 files changed, 26 insertions(+), 54 deletions(-) diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index 136bf654..dff2cbaf 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use super::{DefaultLibraryLoader, LibraryLoader, LibraryProvider}; +use super::{BlockchainLibraryLoader, LibraryLoader, LibraryProvider}; use crate::client::TonClient; use crate::contract::{TonContractError, TonContractFactory}; @@ -26,7 +26,7 @@ impl TonContractFactoryBuilder { const DEFAULT_PRESYNC_BLOCKS: i32 = 50; pub(crate) fn new(client: &TonClient) -> Self { - let loader = DefaultLibraryLoader::new(client); + let loader = BlockchainLibraryLoader::new(client); let library_provider = LibraryProvider::new(loader); TonContractFactoryBuilder { client: client.clone(), diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index 0c5b7bb1..626414f0 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -16,53 +16,47 @@ pub struct ContractLibraryDict { #[async_trait] pub trait LibraryLoader: Send + Sync { - async fn get_library(&self, hash: &TonHash) -> Result, TonLibraryError>; - - async fn get_libraries(&self, hashes: &[TonHash]) -> Result, TonLibraryError>; + async fn load_libraries( + &self, + hashes: &[TonHash], + seqno: Option, + ) -> Result, TonLibraryError>; } -pub struct DefaultLibraryLoader { +pub struct BlockchainLibraryLoader { client: TonClient, } -impl DefaultLibraryLoader { +impl BlockchainLibraryLoader { pub fn new(client: &TonClient) -> Arc { - Arc::new(DefaultLibraryLoader { + Arc::new(BlockchainLibraryLoader { client: client.clone(), }) } } #[async_trait] -impl LibraryLoader for DefaultLibraryLoader { - async fn get_library(&self, hash: &TonHash) -> Result, TonLibraryError> { - let library_result = self.get_libraries(&[*hash]).await?; - match library_result.len() { - 0 => { - log::warn!("Library not found for {:?}", hash); - Ok(None) - } - 1 => Ok(Some(library_result[0].clone())), - _ => Err(TonLibraryError::MultipleLibrariesReturned), - } - } - - async fn get_libraries(&self, hashes: &[TonHash]) -> Result, TonLibraryError> { +impl LibraryLoader for BlockchainLibraryLoader { + async fn load_libraries( + &self, + hashes: &[TonHash], + _seqno: Option, + ) -> Result, TonLibraryError> { let mut results = Vec::new(); // If hashes exceed MAX_LIBS_REQUESTED, split them into chunks for chunk in hashes.chunks(Self::MAX_LIBS_REQUESTED) { - let mut partial_result = self.get_libraries_impl(chunk).await?; + let mut partial_result = self.load_libraries_impl(chunk).await?; results.append(&mut partial_result); } Ok(results) } } -impl DefaultLibraryLoader { +impl BlockchainLibraryLoader { const MAX_LIBS_REQUESTED: usize = 255; - async fn get_libraries_impl( + async fn load_libraries_impl( &self, hashes: &[TonHash], ) -> Result, TonLibraryError> { diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 90688c24..0bce0d6c 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -51,13 +51,14 @@ impl LibraryProvider { pub async fn get_libs_dict<'a, I>( &self, cells: I, + seqno: Option, ) -> Result where I: IntoIterator, { let refs = LibraryProvider::extract_library_hashes(cells)?; - let libs = self.loader.get_libraries(refs.as_slice()).await?; + let libs = self.loader.load_libraries(refs.as_slice(), seqno).await?; let lib_hashmap = libs.into_iter().map(|l| (l.cell_hash(), l)).collect(); diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index 91c41dff..8a258020 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -102,7 +102,7 @@ impl TonContractState { let libs = self .factory .library_provider() - .get_libs_dict(&[code, data]) + .get_libs_dict(&[code, data], Some(self.account_state.block_id.seqno)) .await?; let run_result = unsafe { diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 86126524..8aa26b56 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -244,7 +244,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { )?; let libs = factory .library_provider() - .get_libs_dict(&[code_cell, data_cell]) + .get_libs_dict(&[code_cell, data_cell], None) .await?; let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( diff --git a/client/tests/library_test.rs b/client/tests/library_test.rs index 6c208cd2..9aeb24f9 100644 --- a/client/tests/library_test.rs +++ b/client/tests/library_test.rs @@ -2,36 +2,13 @@ use std::str::FromStr; use tokio_test::assert_ok; use tonlib_client::contract::{ - DefaultLibraryLoader, LibraryLoader, LibraryProvider, TonContractFactory, + BlockchainLibraryLoader, LibraryLoader, LibraryProvider, TonContractFactory, }; use tonlib_core::cell::BagOfCells; use tonlib_core::{TonAddress, TonHash}; mod common; -#[tokio::test] -async fn test_get_lib_by_hash() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - - let expected_lib_id = TonHash::from([ - 159, 49, 244, 244, 19, 163, 172, 203, 112, 108, 136, 150, 42, 198, 157, 89, 16, 59, 1, 58, - 10, 221, 207, 174, 237, 93, 215, 60, 24, 250, 152, 168, - ]); - log::info!("{:?}", expected_lib_id); - - let library_loader = DefaultLibraryLoader::new(&client); - let maybe_lib = library_loader.get_library(&expected_lib_id).await?; - assert!(maybe_lib.is_some()); - - let lib = maybe_lib.unwrap(); - let lib_hash = lib.cell_hash(); - - assert_eq!(expected_lib_id, lib_hash); - - Ok(()) -} - #[tokio::test] async fn test_get_libs_by_hash() -> anyhow::Result<()> { common::init_logging(); @@ -43,9 +20,9 @@ async fn test_get_libs_by_hash() -> anyhow::Result<()> { ]); log::info!("{:?}", expected_lib_id); - let library_loader = DefaultLibraryLoader::new(&client); + let library_loader = BlockchainLibraryLoader::new(&client); let maybe_lib = library_loader - .get_libraries(&[expected_lib_id, expected_lib_id]) + .load_libraries(&[expected_lib_id, expected_lib_id], None) .await?; assert_eq!(maybe_lib.len(), 1); From 201f7f98dbe862b95ff67f27b52db3ee37d8a5a3 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Sun, 22 Dec 2024 14:09:02 +0000 Subject: [PATCH 04/18] Downstream 0.22.4 --- CHANGELOG.md | 28 ++- .../src/contract/nft/collection_contract.rs | 2 +- client/src/contract/nft/item_contract.rs | 3 +- client/tests/meta_loader_test.rs | 135 ++++++++++++++ client/tests/nft_test.rs | 172 ++++++------------ 5 files changed, 222 insertions(+), 118 deletions(-) create mode 100644 client/tests/meta_loader_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 40c20581..a85988b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,31 +1,45 @@ ### v0.19.3 + * Resolve be-1426 "use lite to adjust init block" * Ni: add-get-account-state-by-transaction * Impl #be-1317: serialize_dict * Impl #be-1412: partialeq for invalidmesage + ### v0.20.0 + * Impl #be-1604: update_recent_init_block on tonlib client init * Impl #be-1441: improve dict load/store + ### v0.20.1 + * Fix dependency pinning + ### v0.20.2 -* Impl #be-1619: technical release + +* Impl #be-1619: technical release + ### v0.20.3 + * Impl #be-1464: read dict key TonHash in BE format + ### v0.21.0 + ### v0.21.1 + * Impl #BE-1662: Limited get raw txs * Expose emulator_set_verbosity_level, enabling ability to keep application log dry (#115) * Impl #BE-1620: More strict dictionary handling * Impl #BE-1624: Support wallet v5r1 * Impl #BE-1660: made LiteServerInfo fields public * Impl #BE-1386: Fix handling null instead of dict in TvmStackEntry -* Impl #BE-1625: moved tx hash mismatch handling in factory cache +* Impl #BE-1625: moved tx hash mismatch handling in factory cache * Impl #be-1742: fix read_collection_metadata_content for internal * Impl #ni: update tvm integration tests * Impl #ni: tvmemulator cleanup * Bump tonlib-sys to 2024.10.2 + ### v0.22.2 + * Impl #be-1430: config path via environment variable * Impl #be-1708: added cmp for tonaddress * Impl #be-1785: removed state_cache feature @@ -34,11 +48,19 @@ * Ni: added into_single_root method * Ni: bump thiserror dependency version * Iml #be-1761: fix memory leak in emulator + ### v0.22.1 -* added From for [u8; TON_HASH_LEN] + +* added From for [u8; TON_HASH_LEN] + ### v0.22.2 + * Bump tonlib-sys + ### v0.22.3 + * Impl #be-1820: rm unwrap from jettonmetadata conversion + ### v0.22.4 + * make pub TvmEmulatorUnsafe diff --git a/client/src/contract/nft/collection_contract.rs b/client/src/contract/nft/collection_contract.rs index 09bab829..fbd8625a 100644 --- a/client/src/contract/nft/collection_contract.rs +++ b/client/src/contract/nft/collection_contract.rs @@ -13,7 +13,7 @@ use crate::meta::MetaDataContent; use crate::types::TvmStackEntry; /// Data returned by get_collection_data according to TEP-62 -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct NftCollectionData { /// The count of currently deployed NFT items in collection. /// Generally, collection should issue NFT with sequential indexes (see Rationale(2) ). diff --git a/client/src/contract/nft/item_contract.rs b/client/src/contract/nft/item_contract.rs index 5e3f031d..554e4c56 100644 --- a/client/src/contract/nft/item_contract.rs +++ b/client/src/contract/nft/item_contract.rs @@ -12,7 +12,7 @@ use crate::meta::MetaDataContent; use crate::types::TvmStackEntry; /// Data returned by get_static_data according to TEP-62 -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct NftItemData { /// if not zero, then this NFT is fully initialized and ready for interaction. pub init: bool, @@ -81,6 +81,7 @@ pub trait NftItemContract: TonContractInterface { /// the individual content of this NFT item and /// returns the full content of the NFT item in format /// that complies with standard TEP-64. + // TODO: Issue https://github.com/Selarion/tonlib-rs/issues/2 async fn get_nft_content( &self, index: &BigUint, diff --git a/client/tests/meta_loader_test.rs b/client/tests/meta_loader_test.rs new file mode 100644 index 00000000..40cbc375 --- /dev/null +++ b/client/tests/meta_loader_test.rs @@ -0,0 +1,135 @@ +use serde_json::json; +use tokio_test::assert_ok; +use tonlib_client::meta::{ + LoadMeta, MetaDataContent, NftCollectionMetaData, NftColletionMetaLoader, NftItemMetaData, + NftItemMetaLoader, +}; + +mod common; + +// ---- Nft item metadata load tests +#[tokio::test] +async fn test_load_item_metadata() -> anyhow::Result<()> { + common::init_logging(); + let meta_loader = assert_ok!(NftItemMetaLoader::default()); + let content = MetaDataContent::External { + uri: "https://nft.fragment.com/number/88805397120.json".to_string(), + }; + + let expected_res = NftItemMetaData { + name: Some("+888 0539 7120".to_string()), + description: Some( + "The anonymous number +888 0539 7120 that can be used to create a Telegram \ + account that is not tied to a SIM card." + .to_string(), + ), + image: Some("https://nft.fragment.com/number/88805397120.webp".to_string()), + content_url: None, + attributes: None, + }; + + let res = assert_ok!(meta_loader.load(&content).await); + assert_eq!(expected_res, res); + Ok(()) +} + +#[tokio::test] +async fn test_load_item_metadata_arkenston() -> anyhow::Result<()> { + common::init_logging(); + let meta_loader = assert_ok!(NftItemMetaLoader::default()); + let content = MetaDataContent::External { + uri: "https://meta.ston.fi/meta/stake/v1/0:E1477E82F29495B7284E16D13D9E1D8316AE166146D36459AF8DF9D2865AD7AB.json".to_string(), + }; + + let expected_res = NftItemMetaData { + name: Some("ARKENSTON NFT".to_string()), + description: Some("Staked 30.000000000 STON on STON.fi from 04 Sep 2023.".to_string()), + image: Some("https://static.ston.fi/stake-nft/i4.jpg".to_string()), + content_url: None, + attributes: None, + }; + + let res = assert_ok!(meta_loader.load(&content).await); + assert_eq!(expected_res, res); + Ok(()) +} + +#[tokio::test] +async fn test_load_item_metadata_with_attributes() -> anyhow::Result<()> { + common::init_logging(); + let meta_loader = assert_ok!(NftItemMetaLoader::default()); + let content = MetaDataContent::External { + uri: "https://s.getgems.io/nft/c/64284ddbde940b5d6ebc34f8/12/meta.json".to_string(), + }; + + let attributes = json!([ + {"trait_type": "Pikachu","value": "100"}, + {"trait_type": "Pokemon","value": "100"}, + {"trait_type": "cocktail","value": "100"}, + {"trait_type": "rocks","value": "100"} + ]); + let expected_res = NftItemMetaData { + name: Some("Pokemon Pikachu #013 💎".to_string()), + description: Some( + "The legendary Pokemon Pikachu from the exclusive collection. Gather everyone!" + .to_string(), + ), + image: Some("https://s.getgems.io/nft/c/64284ddbde940b5d6ebc34f8/12/image.png".to_string()), + attributes: Some(attributes), + content_url: None, + }; + + let res = assert_ok!(meta_loader.load(&content).await); + assert_eq!(expected_res, res); + Ok(()) +} + +// ---- Nft collection metadata load tests +#[tokio::test] +async fn test_load_collection_metadata_content_arkenston() -> anyhow::Result<()> { + common::init_logging(); + let meta_loader = assert_ok!(NftColletionMetaLoader::default()); + let content = MetaDataContent::External { + uri: "https://meta.ston.fi/meta/stake-collection/v1/0:AC8495DB6DC9FB72F4A468CD19F0DC88FB6A4D0890B945319907E71117E3DAC7.json".to_string() + }; + + let social_links = json!(["https://app.ston.fi/staking"]); + let expected_res = NftCollectionMetaData { + image: Some("https://static.ston.fi/stake-nft/i1.jpg".to_string()), + name: Some("ARKENSTON NFT".to_string()), + description: Some("psSTON STON.fi Stake".to_string()), + social_links: Some(social_links), + marketplace: None, + }; + + let res = assert_ok!(meta_loader.load(&content).await); + assert_eq!(expected_res, res); + Ok(()) +} + +// Note: HTTP-Response to this URI has an additional key "external_link". +// We don't handle it. It's okay. FYI. +#[tokio::test] +async fn test_load_collection_metadata_content() -> anyhow::Result<()> { + common::init_logging(); + let meta_loader = NftColletionMetaLoader::default()?; + let content = MetaDataContent::External { + uri: "https://nft.fragment.com/numbers.json".to_string(), + }; + + let expected_res = NftCollectionMetaData { + image: Some("https://nft.fragment.com/numbers.svg".to_string()), + name: Some("Anonymous Telegram Numbers".to_string()), + description: Some( + "These anonymous numbers can be used to create Telegram accounts \ + that are not tied to SIM cards." + .to_string(), + ), + social_links: None, + marketplace: None, + }; + + let res = assert_ok!(meta_loader.load(&content).await); + assert_eq!(expected_res, res); + Ok(()) +} diff --git a/client/tests/nft_test.rs b/client/tests/nft_test.rs index 6b3df611..08953822 100644 --- a/client/tests/nft_test.rs +++ b/client/tests/nft_test.rs @@ -1,11 +1,15 @@ +use std::str::FromStr; + +use num_bigint::BigUint; use sha2::{Digest, Sha256}; use tokio_test::assert_ok; use tonlib_client::contract::{NftCollectionContract, NftItemContract, TonContractFactory}; -use tonlib_client::meta::{LoadMeta, MetaDataContent, NftColletionMetaLoader, NftItemMetaLoader}; -use tonlib_core::TonHash; +use tonlib_client::meta::MetaDataContent; +use tonlib_core::{TonAddress, TonHash}; mod common; +// ---- Tests methods is work only. #[tokio::test] async fn test_get_nft_data() { common::init_logging(); @@ -18,7 +22,7 @@ async fn test_get_nft_data() { } #[tokio::test] -async fn test_get_nft_collection_data() -> anyhow::Result<()> { +async fn test_get_collection_data() -> anyhow::Result<()> { common::init_logging(); let client = common::new_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; @@ -40,10 +44,9 @@ async fn test_get_nft_address_by_index() -> anyhow::Result<()> { Ok(()) } -// ---------------------nft get item metadata tests - +// ---- Tests methods return valid data. #[tokio::test] -async fn test_get_nft_content_uri() -> anyhow::Result<()> { +async fn test_get_nft_data_is_valid() -> anyhow::Result<()> { common::init_logging(); let client = common::new_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; @@ -51,67 +54,53 @@ async fn test_get_nft_content_uri() -> anyhow::Result<()> { "EQCGZEZZcYO9DK877fJSIEpYMSvfui7zmTXGhq0yq1Ce1Mb6".parse() )); let res = assert_ok!(contract.get_nft_data().await); + + let expected_collection_address = assert_ok!(TonAddress::from_base64_url( + "EQAOQdwdw8kGftJCSFgOErM1mBjYPe4DBPq8-AhF6vr9si5N" + )); + let expected_index = assert_ok!(BigUint::from_str( + "15995005474673311991943775795727481451058346239240361725119718297821926435889", + )); + + assert!(res.init); + assert_eq!(res.index, expected_index); + assert_eq!(res.collection_address, expected_collection_address); assert_eq!( res.individual_content, MetaDataContent::External { - uri: "https://nft.fragment.com/number/88805397120.json".to_string() + uri: "https://nft.fragment.com/number/88805397120.json".to_string(), } ); - let meta_loader = assert_ok!(NftItemMetaLoader::default()); - let content_res = assert_ok!(meta_loader.load(&res.individual_content).await); - assert_eq!( - content_res.name.as_ref().unwrap(), - &String::from("+888 0539 7120") - ); - assert_eq!( - content_res.image.as_ref().unwrap(), - &String::from("https://nft.fragment.com/number/88805397120.webp") - ); - Ok(()) -} -#[tokio::test] -async fn test_get_nft_content_arkenston() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - let factory = assert_ok!(TonContractFactory::builder(&client).build().await); - let contract = factory.get_contract(&assert_ok!( - "EQDhR36C8pSVtyhOFtE9nh2DFq4WYUbTZFmvjfnShlrXq2cz".parse() - )); - let res = assert_ok!(contract.get_nft_data().await); - let meta_loader = assert_ok!(NftItemMetaLoader::default()); - let content_res = assert_ok!(meta_loader.load(&res.individual_content).await); - assert_eq!( - content_res.image.unwrap(), - "https://static.ston.fi/stake-nft/i4.jpg" - ); - assert_eq!(content_res.name.unwrap(), "ARKENSTON NFT"); Ok(()) } #[tokio::test] -async fn test_get_nft_content_some() -> anyhow::Result<()> { +async fn test_get_nft_data_internal() -> anyhow::Result<()> { common::init_logging(); let client = common::new_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; - let contract = factory.get_contract(&assert_ok!( - "EQCiXgoveScGKGGqo50HbmwP3goKJaEfu9QmeBRJ-jbRxM21".parse() - )); - let res = assert_ok!(contract.get_nft_data().await); - let meta_loader = assert_ok!(NftItemMetaLoader::default()); - let content_res = assert_ok!(meta_loader.load(&res.individual_content).await); - assert_eq!( - content_res.image.unwrap(), - "https://mars.tonplanets.com/i/biomes/4v4.jpg" - ); - assert_eq!(content_res.name.unwrap(), "Anda"); + let contract = + factory.get_contract(&"EQDUF9cLVBH3BgziwOAIkezUdmfsDxxJHd6WSv0ChIUXYwCx".parse()?); + let res = contract.get_nft_data().await?; + + let internal = match res.individual_content { + MetaDataContent::Internal { dict } => dict, + _ => panic!("Expected internal content"), + }; + + let expected_key = { + let mut hasher: Sha256 = Sha256::new(); + hasher.update("public_keys"); + let slice = &hasher.finalize()[..]; + TryInto::::try_into(slice)? + }; + assert!(internal.contains_key(&expected_key)); Ok(()) } -// ---------------------nft get collection metadata tests - #[tokio::test] -async fn test_get_nft_collection_content_uri() -> anyhow::Result<()> { +async fn test_get_collection_data_is_valid() -> anyhow::Result<()> { common::init_logging(); let client = common::new_archive_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; @@ -120,83 +109,40 @@ async fn test_get_nft_collection_content_uri() -> anyhow::Result<()> { )); let res = assert_ok!(contract.get_collection_data().await); + assert_eq!(res.next_item_index, -1); assert_eq!( res.collection_content, MetaDataContent::External { - uri: "https://nft.fragment.com/numbers.json".to_string() + uri: "https://nft.fragment.com/numbers.json".to_string(), } ); - - let meta_loader = NftColletionMetaLoader::default()?; - let content_res = assert_ok!(meta_loader.load(&res.collection_content).await); - assert_eq!( - content_res.name.as_ref().unwrap(), - &String::from("Anonymous Telegram Numbers") - ); - assert_eq!( - content_res.image.as_ref().unwrap(), - &String::from("https://nft.fragment.com/numbers.svg") - ); Ok(()) } #[tokio::test] -async fn test_get_nft_collection_content_arkenston() -> anyhow::Result<()> { +async fn test_get_nft_address_by_index_is_valid() -> anyhow::Result<()> { common::init_logging(); let client = common::new_mainnet_client().await; - let factory = assert_ok!(TonContractFactory::builder(&client).build().await); - let contract = factory.get_contract(&assert_ok!( - "EQCshJXbbcn7cvSkaM0Z8NyI-2pNCJC5RTGZB-cRF-Pax1lY".parse() - )); - let res = assert_ok!(contract.get_collection_data().await); - let meta_loader = assert_ok!(NftColletionMetaLoader::default()); - let content_res = assert_ok!(meta_loader.load(&res.collection_content).await); - assert_eq!(content_res.name.unwrap(), "ARKENSTON NFT"); - assert_eq!( - content_res.image.unwrap(), - "https://static.ston.fi/stake-nft/i1.jpg" - ); - Ok(()) -} - -#[tokio::test] -async fn test_get_nft_collection_content_some() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - let factory = assert_ok!(TonContractFactory::builder(&client).build().await); + let factory = TonContractFactory::builder(&client).build().await?; let contract = factory.get_contract(&assert_ok!( - "EQCbOjwru5tBb2aaXZEHbiTCVIYQ6yDNAe8SSZkP4CozibHM".parse() + "EQB2iHQ9lmJ9zvYPauxN9hVOfHL3c_fuN5AyRq5Pm84UH6jC".parse() )); - let res = assert_ok!(contract.get_nft_data().await); - let meta_loader = assert_ok!(NftColletionMetaLoader::default()); - let content_res = assert_ok!(meta_loader.load(&res.individual_content).await); - assert_eq!(content_res.name.unwrap(), "Pokemon Pikachu #013 💎"); - assert_eq!( - content_res.image.unwrap(), - "https://s.getgems.io/nft/c/64284ddbde940b5d6ebc34f8/12/image.png" - ); - Ok(()) -} -#[tokio::test] -async fn test_get_nft_content_external() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - let factory = TonContractFactory::builder(&client).build().await?; - let contract = - factory.get_contract(&"EQDUF9cLVBH3BgziwOAIkezUdmfsDxxJHd6WSv0ChIUXYwCx".parse()?); - let nft_data = contract.get_nft_data().await?; - let internal = match nft_data.individual_content { - MetaDataContent::Internal { dict } => dict, - _ => panic!("Expected internal content"), - }; + let res_0 = assert_ok!(contract.get_nft_address_by_index(0).await); + let res_2 = assert_ok!(contract.get_nft_address_by_index(2).await); + let res_1 = assert_ok!(contract.get_nft_address_by_index(1).await); - let expected_key = { - let mut hasher: Sha256 = Sha256::new(); - hasher.update("public_keys"); - let slice = &hasher.finalize()[..]; - TryInto::::try_into(slice)? - }; - assert!(internal.contains_key(&expected_key)); + let expected_addr_0 = assert_ok!(TonAddress::from_base64_url( + "EQBKwtMZSZurMxGp7FLZ_lM9t54_ECEsS46NLR3qfIwwTnKW" + )); + let expected_addr_1 = assert_ok!(TonAddress::from_base64_url( + "EQB6rnPIZr8dXmLy0xVp4lTe1AlYRwOUghEG9zzCcCkCp8IS" + )); + let expected_addr_2 = assert_ok!(TonAddress::from_base64_url( + "EQD0VQNu41wZmWMQjXfifnljGR0vOAULh0stBLItskMavwH0" + )); + assert_eq!(res_0, expected_addr_0); + assert_eq!(res_1, expected_addr_1); + assert_eq!(res_2, expected_addr_2); Ok(()) } From e14ca137be526cc9471433f763e9e88c2b06995b Mon Sep 17 00:00:00 2001 From: Dmitrii Korchagin Date: Wed, 25 Dec 2024 15:47:51 +0100 Subject: [PATCH 05/18] Impl #NI: try fix another leak --- client/src/contract/state.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index 8a258020..f26acabd 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -201,6 +201,9 @@ impl TonContractState { .smc_run_get_method(state.id, &method.into(), &stack_tl) .await?; + // TODO make it properly using drop! + tokio::spawn(async move { state.conn.smc_forget(state.id).await }); + let stack = run_result .stack .elements From fa7aa72cd33529187c565cb0e28cd790b3b95ac8 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 2 Jan 2025 13:31:41 +0000 Subject: [PATCH 06/18] Impl #BE-1881: Replaced dashmap by tokio::sync:Mutex to avoid mem leaks --- Cargo.toml | 1 - client/Cargo.toml | 1 - client/src/client/connection.rs | 21 ++++++++++++--------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6d3a7503..81a18f66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ base64 = "0.22" base64-serde = "0.7" bitstream-io = "2.2" crc = "3" -dashmap = "5" futures = "0.3" hex = "0.4" hmac = { version = "0.12", features = ["std"] } diff --git a/client/Cargo.toml b/client/Cargo.toml index 25497ea5..93137eef 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -29,7 +29,6 @@ base64.workspace = true base64-serde.workspace = true crc.workspace = true -dashmap.workspace = true futures.workspace = true hex.workspace = true diff --git a/client/src/client/connection.rs b/client/src/client/connection.rs index 5a390737..6d6e76c0 100644 --- a/client/src/client/connection.rs +++ b/client/src/client/connection.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Weak}; use std::thread; @@ -5,8 +6,7 @@ use std::thread::JoinHandle; use std::time::Instant; use async_trait::async_trait; -use dashmap::DashMap; -use tokio::sync::{broadcast, oneshot, Semaphore, SemaphorePermit}; +use tokio::sync::{broadcast, oneshot, Mutex, Semaphore, SemaphorePermit}; use crate::client::{ TonClientError, TonClientInterface, TonConnectionCallback, TonConnectionParams, @@ -28,7 +28,7 @@ struct RequestData { sender: oneshot::Sender>, } -type RequestMap = DashMap; +type RequestMap = Mutex>; type TonNotificationSender = broadcast::Sender>; struct Inner { @@ -85,10 +85,12 @@ impl TonConnection { } else { None }; + + let request_map = Mutex::new(HashMap::new()); let inner = Inner { tl_client: TlTonClient::new(tag.as_str()), counter: AtomicU32::new(0), - request_map: RequestMap::new(), + request_map, notification_sender: sender, callback, _notification_receiver: receiver, @@ -271,14 +273,14 @@ impl TonClientInterface for TonConnection { send_time: Instant::now(), sender: tx, }; - self.inner.request_map.insert(cnt, data); + self.inner.request_map.lock().await.insert(cnt, data); self.inner .callback .on_invoke(self.inner.tl_client.get_tag(), cnt, function); let res = self.inner.tl_client.send(function, extra.as_str()); if let Err(e) = res { - let (_, data) = self.inner.request_map.remove(&cnt).unwrap(); + let data = self.inner.request_map.lock().await.remove(&cnt).unwrap(); let tag = self.inner.tl_client.get_tag(); let duration = Instant::now().duration_since(data.send_time); let res = Err(TonClientError::TlError(e)); @@ -322,12 +324,13 @@ fn run_loop(tag: String, weak_inner: Weak, callback: Arc = match ton_result { Ok(TonResult::Error { code, message }) => { let method = maybe_data .as_ref() - .map(|d| d.1.method) + .map(|d| d.method) .unwrap_or(NOT_AVAILABLE); Err(TonClientError::TonlibError { method, @@ -339,7 +342,7 @@ fn run_loop(tag: String, weak_inner: Weak, callback: Arc Ok(r), }; - if let Some((_, data)) = maybe_data { + if let Some(data) = maybe_data { // Found corresponding request, reply to it let request_id = maybe_request_id.unwrap(); // Can't be empty if data is not empty let now = Instant::now(); From 0d6c8339ce0a0ced37e30ede32a23f425193a005 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Fri, 3 Jan 2025 09:28:34 +0000 Subject: [PATCH 07/18] Impl #BE-1846 : library cache --- client/src/contract/factory.rs | 3 + client/src/contract/factory/builder.rs | 12 +- client/src/contract/factory/cache.rs | 19 +- client/src/contract/factory/library_loader.rs | 24 ++- .../src/contract/factory/library_provider.rs | 177 ++++++++++++++++-- client/src/contract/state.rs | 2 +- client/tests/contract_emulator_test.rs | 55 +++++- 7 files changed, 268 insertions(+), 24 deletions(-) diff --git a/client/src/contract/factory.rs b/client/src/contract/factory.rs index bbbf346e..61ff18b1 100644 --- a/client/src/contract/factory.rs +++ b/client/src/contract/factory.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -44,6 +45,7 @@ impl TonContractFactory { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, + current_seqno: Arc, ) -> Result { let cache = if with_cache { let cache = ContractFactoryCache::new( @@ -53,6 +55,7 @@ impl TonContractFactory { txid_cache_capacity, txid_cache_time_to_live, presync_blocks, + current_seqno, ) .await?; Some(cache) diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index dff2cbaf..9819d856 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -14,6 +15,7 @@ pub struct TonContractFactoryBuilder { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, + current_seqno: Arc, } impl TonContractFactoryBuilder { @@ -26,17 +28,19 @@ impl TonContractFactoryBuilder { const DEFAULT_PRESYNC_BLOCKS: i32 = 50; pub(crate) fn new(client: &TonClient) -> Self { + let current_seqno_counter: Arc = Arc::new(0.into()); let loader = BlockchainLibraryLoader::new(client); - let library_provider = LibraryProvider::new(loader); + let library_provider = LibraryProvider::new(loader, None, current_seqno_counter.clone()); TonContractFactoryBuilder { client: client.clone(), - with_cache: false, + with_cache: true, account_state_cache_capacity: 0, account_state_cache_time_to_live: Duration::default(), txid_cache_capacity: 0, txid_cache_time_to_live: Duration::default(), presync_blocks: Self::DEFAULT_PRESYNC_BLOCKS, library_provider, + current_seqno: current_seqno_counter, } } @@ -79,11 +83,13 @@ impl TonContractFactoryBuilder { self.txid_cache_time_to_live, self.presync_blocks, self.library_provider.clone(), + self.current_seqno.clone(), ) .await } pub fn with_library_loader(mut self, library_loader: &Arc) -> Self { - let library_provider = LibraryProvider::new(library_loader.clone()); + let library_provider = + LibraryProvider::new(library_loader.clone(), None, self.current_seqno.clone()); self.library_provider = library_provider; self } diff --git a/client/src/contract/factory/cache.rs b/client/src/contract/factory/cache.rs index 16103101..bc8e62ba 100644 --- a/client/src/contract/factory/cache.rs +++ b/client/src/contract/factory/cache.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -33,6 +33,7 @@ impl ContractFactoryCache { txid_cache_capacity: u64, txid_state_cache_time_to_live: Duration, presync_blocks: i32, + current_seqno: Arc, ) -> Result { let inner = Inner { client: client.clone(), @@ -49,6 +50,7 @@ impl ContractFactoryCache { account_state_cache_counters: ContractFactoryCacheCounters::default(), tx_id_cache_counters: ContractFactoryCacheCounters::default(), + current_seqno, }; let arc_inner = Arc::new(inner); @@ -137,6 +139,7 @@ impl ContractFactoryCache { match masterchain_info_result { Ok((_, info)) => { let first_block_seqno = info.last.seqno - inner.presync_blocks; + let block_stream = BlockStream::new(client, first_block_seqno); break block_stream; } @@ -161,10 +164,16 @@ impl ContractFactoryCache { if weak_inner.upgrade().is_none() { break; } - let block_result = block_stream.next().await; let block = match block_result { - Ok(block) => block, + Ok(block) => { + if let Some(inner) = weak_inner.upgrade() { + inner + .current_seqno + .store(block.master_shard.seqno, Ordering::Relaxed) + }; + block + } Err(e) => { log::warn!( "[ContractFactoryCache] Could not retrieve next block: {:?}", @@ -177,6 +186,9 @@ impl ContractFactoryCache { loop { if let Some(inner) = weak_inner.upgrade() { + inner + .current_seqno + .store(block.master_shard.seqno, Ordering::Relaxed); let process_result = inner.process_next_block(&block).await; match process_result { Ok(_) => break, @@ -227,6 +239,7 @@ struct Inner { presync_blocks: i32, tx_id_cache_counters: ContractFactoryCacheCounters, account_state_cache_counters: ContractFactoryCacheCounters, + current_seqno: Arc, } impl Inner { diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index 626414f0..df89a307 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -1,17 +1,35 @@ +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use tonlib_core::cell::{ArcCell, BagOfCells}; +use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; +use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder, TonCellError}; use tonlib_core::TonHash; use crate::client::{TonClient, TonClientInterface}; use crate::contract::TonLibraryError; use crate::tl::TonLibraryId; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct ContractLibraryDict { pub dict_boc: Vec, - pub keys: Vec, + pub keys: Vec, +} + +impl TryFrom> for ContractLibraryDict { + type Error = TonCellError; + + fn try_from(value: HashMap) -> Result { + let keys = value.keys().copied().collect(); + let lib_cell = CellBuilder::new() + .store_dict_data(256, val_writer_ref_cell, value)? + .build()?; + + let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; + + let dict = ContractLibraryDict { dict_boc, keys }; + Ok(dict) + } } #[async_trait] diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 0bce0d6c..999f611d 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -1,6 +1,10 @@ -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; +use std::time::Duration; +use futures::future::join_all; +use moka::future::Cache; use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; use tonlib_core::TonHash; @@ -8,14 +12,47 @@ use tonlib_core::TonHash; use super::{ContractLibraryDict, LibraryLoader}; use crate::contract::TonLibraryError; +const DEFAULT_LIBRARY_CACHE_CAPACITY: u64 = 300; +const DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); +const DEFAULT_INTERNAL_CACHE_CAPACITY: u64 = 300; +const DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); + +const DEFAULT_LIBRARY_CACHE_PARAMS: LibraryCacheParams = LibraryCacheParams { + lib_cache_capacity: DEFAULT_LIBRARY_CACHE_CAPACITY, + lib_cache_time_to_live: DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE, + internal_cache_capacity: DEFAULT_INTERNAL_CACHE_CAPACITY, + internal_cache_time_to_live: DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE, +}; + +#[derive(Clone, Copy)] +pub struct LibraryCacheParams { + lib_cache_capacity: u64, + lib_cache_time_to_live: Duration, + internal_cache_capacity: u64, + internal_cache_time_to_live: Duration, +} + #[derive(Clone)] pub struct LibraryProvider { + inner: Arc, +} + +struct Inner { loader: Arc, + cache_by_hash_seqno: Cache<(TonHash, i32), Option>, + cache_by_hash: Cache, + current_seqno: Arc, } impl LibraryProvider { - pub fn new(loader: Arc) -> Self { - Self { loader } + pub fn new( + loader: Arc, + cache_params: Option, + current_seqno: Arc, + ) -> Self { + Self { + inner: Arc::new(Inner::new(loader, cache_params, current_seqno)), + } } pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> @@ -48,29 +85,145 @@ impl LibraryProvider { Ok(lib_hashes) } - pub async fn get_libs_dict<'a, I>( + pub async fn get_libs_by_seqno<'a, I>( &self, cells: I, - seqno: Option, + seqno: i32, ) -> Result where I: IntoIterator, { - let refs = LibraryProvider::extract_library_hashes(cells)?; - - let libs = self.loader.load_libraries(refs.as_slice(), seqno).await?; + let lib_hashes = LibraryProvider::extract_library_hashes(cells)?; + let libs = self.inner.get_libs(lib_hashes.as_slice(), seqno).await?; + LibraryProvider::store_to_dict(lib_hashes.as_slice(), libs) + } - let lib_hashmap = libs.into_iter().map(|l| (l.cell_hash(), l)).collect(); + pub async fn get_libs_latest<'a, I>( + &self, + cells: I, + ) -> Result + where + I: IntoIterator, + { + let seqno = self.inner.current_seqno.load(Ordering::Relaxed); + self.get_libs_by_seqno(cells, seqno).await + } + fn store_to_dict( + library_hashes: &[TonHash], + lib_hashmap: HashMap, + ) -> Result { let lib_cell = CellBuilder::new() .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? .build()?; let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - - let keys = refs.iter().map(|r| (*r).into()).collect(); - + let keys = library_hashes.to_vec(); let dict = ContractLibraryDict { dict_boc, keys }; Ok(dict) } } + +impl Inner { + pub(crate) fn new( + loader: Arc, + cache_params: Option, + current_seqno: Arc, + ) -> Self { + let cache_params = cache_params.unwrap_or(DEFAULT_LIBRARY_CACHE_PARAMS); + + let library_cache = Cache::builder() + .max_capacity(cache_params.lib_cache_capacity) + .time_to_live(cache_params.lib_cache_time_to_live) + .build(); + let internal_cache = Cache::builder() + .max_capacity(cache_params.internal_cache_capacity) + .time_to_live(cache_params.internal_cache_time_to_live) + .build(); + Self { + loader, + cache_by_hash_seqno: library_cache, + cache_by_hash: internal_cache, + current_seqno, + } + } + + pub(crate) async fn get_libs( + &self, + lib_hashes: &[TonHash], + seqno: i32, + ) -> Result, TonLibraryError> +where { + let mut result_libs = HashMap::new(); + + let keys = lib_hashes + .iter() + .map(|h| (*h, seqno)) + .collect::>(); + let cached_libs_future = keys + .iter() + .map(|key| async move { (key.0, self.cache_by_hash_seqno.get(key).await) }); + let maybe_cached_libs = join_all(cached_libs_future).await; + + let mut hashes_to_load = vec![]; + for (hash, value) in maybe_cached_libs { + // outer option means whether library is in the cache or not + // inner option means whether library is available at certain seqno + match value { + Some(Some(lib)) => { + result_libs.insert(hash, lib); + } + Some(None) => {} + None => { + log::trace!("loading lib from BC: {:?}", hash); + hashes_to_load.push(hash); + } + } + } + + // load libs from blockchain + let mut blockchain_libs = self + .loader + .load_libraries(hashes_to_load.as_slice(), Some(seqno)) + .await?; + self.replace_by_existing_data(&mut blockchain_libs).await; + result_libs.extend(blockchain_libs.iter().map(|l| (l.cell_hash(), l.clone()))); + + self.insert_to_lib_cache(blockchain_libs, seqno).await?; + Ok(result_libs) + } + + async fn insert_to_lib_cache( + &self, + libs: Vec, + seqno: i32, + ) -> Result<(), TonLibraryError> { + let mut cache_insert_futures = Vec::with_capacity(libs.len()); + let mut internal_cache_insert_futures = Vec::with_capacity(libs.len()); + for lib in libs { + cache_insert_futures.push( + self.cache_by_hash_seqno + .insert((lib.cell_hash(), seqno), Some(lib.clone())), + ); + internal_cache_insert_futures.push(self.cache_by_hash.insert(lib.cell_hash(), lib)); + } + + join_all(cache_insert_futures).await; + join_all(internal_cache_insert_futures).await; + Ok(()) + } + + async fn replace_by_existing_data(&self, loaded_libs: &mut [ArcCell]) { + let loaded_libs_keys: Vec<_> = loaded_libs.iter().map(|l| l.cell_hash()).collect(); + let future_internals = loaded_libs_keys.iter().map(|k| self.cache_by_hash.get(k)); + + let internal_cached_libs = join_all(future_internals).await; + + // replace loaded libs from blockchain to ones from internal cache. + for i in 0..loaded_libs.len() { + if let Some(lib) = &internal_cached_libs[i] { + loaded_libs[i] = lib.clone() + } + } + } +} diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index f26acabd..f6534490 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -102,7 +102,7 @@ impl TonContractState { let libs = self .factory .library_provider() - .get_libs_dict(&[code, data], Some(self.account_state.block_id.seqno)) + .get_libs_latest(&[code, data]) .await?; let run_result = unsafe { diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 8aa26b56..ab46eea4 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -114,7 +114,7 @@ async fn test_emulator_get_wallet_address() { #[tokio::test] async fn test_emulate_ston_router_v2() -> anyhow::Result<()> { common::init_logging(); - let client = common::new_mainnet_client().await; + let client = common::new_archive_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; @@ -244,7 +244,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { )?; let libs = factory .library_provider() - .get_libs_dict(&[code_cell, data_cell], None) + .get_libs_by_seqno(&[code_cell, data_cell], 47497716) .await?; let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( @@ -330,3 +330,54 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_lib_cache_works() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + let factory = TonContractFactory::builder(&client) + .with_default_cache() + .build() + .await?; + + let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; + + let contract = factory.get_contract(&router_address); + let state = contract.get_state().await?; + + let code = state.get_account_state().code.clone(); + let data = state.get_account_state().data.clone(); + + let code_cell = BagOfCells::parse(&code)?.into_single_root()?; + let data_cell = BagOfCells::parse(&data)?.into_single_root()?; + + const MAX_ITER: usize = 100000; + + let mut all_libs = vec![]; + for i in 0..MAX_ITER { + let t = Instant::now(); + let libs = factory + .library_provider() + .get_libs_latest(&[code_cell.clone(), data_cell.clone()]) + .await?; + + if t.elapsed() > Duration::from_millis(10) { + log::info!("iteraion: {}DT {:?}", i, t.elapsed()); + } + + tokio::time::sleep(Duration::from_millis(10)).await; + + all_libs.push(libs) + } + + let bc_lib = &all_libs[0].dict_boc; + let cache_lib = &all_libs[1].dict_boc; + + let bc_cell = BagOfCells::parse(bc_lib)?.into_single_root()?; + + let cache_cell = BagOfCells::parse(cache_lib)?.into_single_root()?; + + assert_eq!(bc_cell, cache_cell); + + Ok(()) +} From 672bf3d12ed0fe3aeedc22ff89897d886a97fd3c Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 14:15:59 +0400 Subject: [PATCH 08/18] retrying invokes on connection --- client/src/client/connection.rs | 36 ++++++++++++++++++++++++++------- client/src/client/types.rs | 7 +++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/client/src/client/connection.rs b/client/src/client/connection.rs index 6d6e76c0..bbb47b05 100644 --- a/client/src/client/connection.rs +++ b/client/src/client/connection.rs @@ -7,7 +7,10 @@ use std::time::Instant; use async_trait::async_trait; use tokio::sync::{broadcast, oneshot, Mutex, Semaphore, SemaphorePermit}; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::RetryIf; +use super::{retry_condition, RetryStrategy}; use crate::client::{ TonClientError, TonClientInterface, TonConnectionCallback, TonConnectionParams, TonNotificationReceiver, @@ -39,6 +42,7 @@ struct Inner { callback: Arc, _notification_receiver: TonNotificationReceiver, semaphore: Option, + retry_strategy: RetryStrategy, } pub struct TonConnection { @@ -86,6 +90,8 @@ impl TonConnection { None }; + let retry_strategy = params.retry_strategy.clone(); + let request_map = Mutex::new(HashMap::new()); let inner = Inner { tl_client: TlTonClient::new(tag.as_str()), @@ -95,6 +101,7 @@ impl TonConnection { callback, _notification_receiver: receiver, semaphore, + retry_strategy, }; let inner_arc = Arc::new(inner); let inner_weak: Weak = Arc::downgrade(&inner_arc); @@ -252,15 +259,16 @@ impl TonConnection { None }) } -} - -#[async_trait] -impl TonClientInterface for TonConnection { - async fn get_connection(&self) -> Result { - Ok(self.clone()) + async fn retrying_invoke( + &self, + function: &TonFunction, + ) -> Result<(TonConnection, TonResult), TonClientError> { + let fi = FixedInterval::from_millis(self.inner.retry_strategy.interval_ms); + let strategy = fi.take(self.inner.retry_strategy.max_retries); + RetryIf::spawn(strategy, || self.do_invoke(function), retry_condition).await } - async fn invoke_on_connection( + async fn do_invoke( &self, function: &TonFunction, ) -> Result<(TonConnection, TonResult), TonClientError> { @@ -302,6 +310,20 @@ impl TonClientInterface for TonConnection { } } +#[async_trait] +impl TonClientInterface for TonConnection { + async fn get_connection(&self) -> Result { + Ok(self.clone()) + } + + async fn invoke_on_connection( + &self, + function: &TonFunction, + ) -> Result<(TonConnection, TonResult), TonClientError> { + self.retrying_invoke(function).await + } +} + impl Clone for TonConnection { fn clone(&self) -> Self { let inner = self.inner.clone(); diff --git a/client/src/client/types.rs b/client/src/client/types.rs index b6f4d9c1..39ee72b8 100644 --- a/client/src/client/types.rs +++ b/client/src/client/types.rs @@ -52,6 +52,8 @@ pub struct TonConnectionParams { pub concurrency_limit: usize, #[serde(default = "default_update_init_block")] pub update_init_block: bool, + #[serde(default = "default_retry_strategy")] + pub retry_strategy: RetryStrategy, } impl Default for TonConnectionParams { @@ -65,6 +67,7 @@ impl Default for TonConnectionParams { notification_queue_length: DEFAULT_NOTIFICATION_QUEUE_LENGTH, concurrency_limit: DEFAULT_CONNECTION_CONCURRENCY_LIMIT, update_init_block: DEFAULT_UPDATE_INIT_BLOCK, + retry_strategy: RetryStrategy::default(), } } } @@ -99,6 +102,10 @@ impl Default for RetryStrategy { } } +fn default_retry_strategy() -> RetryStrategy { + RetryStrategy::default() +} + lazy_static! { pub static ref DEFAULT_RETRY_STRATEGY: RetryStrategy = RetryStrategy::default(); } From f454e75954c53c4e67ff5206e2331fa9dd7b78ea Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 14:17:14 +0400 Subject: [PATCH 09/18] Revert "Merge branch 'main' of ssh://git.stonfi.net:2222/ston-fi/rust/tonlib-rs into BE-1879" This reverts commit 113ab4a38d41a6a184c787e702fb72bcc3943b82, reversing changes made to 672bf3d12ed0fe3aeedc22ff89897d886a97fd3c. --- client/src/contract/factory.rs | 3 - client/src/contract/factory/builder.rs | 12 +- client/src/contract/factory/cache.rs | 19 +- client/src/contract/factory/library_loader.rs | 24 +-- .../src/contract/factory/library_provider.rs | 177 ++---------------- client/src/contract/state.rs | 2 +- client/tests/contract_emulator_test.rs | 55 +----- 7 files changed, 24 insertions(+), 268 deletions(-) diff --git a/client/src/contract/factory.rs b/client/src/contract/factory.rs index 61ff18b1..bbbf346e 100644 --- a/client/src/contract/factory.rs +++ b/client/src/contract/factory.rs @@ -1,4 +1,3 @@ -use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -45,7 +44,6 @@ impl TonContractFactory { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, - current_seqno: Arc, ) -> Result { let cache = if with_cache { let cache = ContractFactoryCache::new( @@ -55,7 +53,6 @@ impl TonContractFactory { txid_cache_capacity, txid_cache_time_to_live, presync_blocks, - current_seqno, ) .await?; Some(cache) diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index 9819d856..dff2cbaf 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -1,4 +1,3 @@ -use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -15,7 +14,6 @@ pub struct TonContractFactoryBuilder { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, - current_seqno: Arc, } impl TonContractFactoryBuilder { @@ -28,19 +26,17 @@ impl TonContractFactoryBuilder { const DEFAULT_PRESYNC_BLOCKS: i32 = 50; pub(crate) fn new(client: &TonClient) -> Self { - let current_seqno_counter: Arc = Arc::new(0.into()); let loader = BlockchainLibraryLoader::new(client); - let library_provider = LibraryProvider::new(loader, None, current_seqno_counter.clone()); + let library_provider = LibraryProvider::new(loader); TonContractFactoryBuilder { client: client.clone(), - with_cache: true, + with_cache: false, account_state_cache_capacity: 0, account_state_cache_time_to_live: Duration::default(), txid_cache_capacity: 0, txid_cache_time_to_live: Duration::default(), presync_blocks: Self::DEFAULT_PRESYNC_BLOCKS, library_provider, - current_seqno: current_seqno_counter, } } @@ -83,13 +79,11 @@ impl TonContractFactoryBuilder { self.txid_cache_time_to_live, self.presync_blocks, self.library_provider.clone(), - self.current_seqno.clone(), ) .await } pub fn with_library_loader(mut self, library_loader: &Arc) -> Self { - let library_provider = - LibraryProvider::new(library_loader.clone(), None, self.current_seqno.clone()); + let library_provider = LibraryProvider::new(library_loader.clone()); self.library_provider = library_provider; self } diff --git a/client/src/contract/factory/cache.rs b/client/src/contract/factory/cache.rs index bc8e62ba..16103101 100644 --- a/client/src/contract/factory/cache.rs +++ b/client/src/contract/factory/cache.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::atomic::{AtomicI32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -33,7 +33,6 @@ impl ContractFactoryCache { txid_cache_capacity: u64, txid_state_cache_time_to_live: Duration, presync_blocks: i32, - current_seqno: Arc, ) -> Result { let inner = Inner { client: client.clone(), @@ -50,7 +49,6 @@ impl ContractFactoryCache { account_state_cache_counters: ContractFactoryCacheCounters::default(), tx_id_cache_counters: ContractFactoryCacheCounters::default(), - current_seqno, }; let arc_inner = Arc::new(inner); @@ -139,7 +137,6 @@ impl ContractFactoryCache { match masterchain_info_result { Ok((_, info)) => { let first_block_seqno = info.last.seqno - inner.presync_blocks; - let block_stream = BlockStream::new(client, first_block_seqno); break block_stream; } @@ -164,16 +161,10 @@ impl ContractFactoryCache { if weak_inner.upgrade().is_none() { break; } + let block_result = block_stream.next().await; let block = match block_result { - Ok(block) => { - if let Some(inner) = weak_inner.upgrade() { - inner - .current_seqno - .store(block.master_shard.seqno, Ordering::Relaxed) - }; - block - } + Ok(block) => block, Err(e) => { log::warn!( "[ContractFactoryCache] Could not retrieve next block: {:?}", @@ -186,9 +177,6 @@ impl ContractFactoryCache { loop { if let Some(inner) = weak_inner.upgrade() { - inner - .current_seqno - .store(block.master_shard.seqno, Ordering::Relaxed); let process_result = inner.process_next_block(&block).await; match process_result { Ok(_) => break, @@ -239,7 +227,6 @@ struct Inner { presync_blocks: i32, tx_id_cache_counters: ContractFactoryCacheCounters, account_state_cache_counters: ContractFactoryCacheCounters, - current_seqno: Arc, } impl Inner { diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index df89a307..626414f0 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -1,35 +1,17 @@ -use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; -use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder, TonCellError}; +use tonlib_core::cell::{ArcCell, BagOfCells}; use tonlib_core::TonHash; use crate::client::{TonClient, TonClientInterface}; use crate::contract::TonLibraryError; use crate::tl::TonLibraryId; -#[derive(Debug, PartialEq)] +#[derive(Debug)] pub struct ContractLibraryDict { pub dict_boc: Vec, - pub keys: Vec, -} - -impl TryFrom> for ContractLibraryDict { - type Error = TonCellError; - - fn try_from(value: HashMap) -> Result { - let keys = value.keys().copied().collect(); - let lib_cell = CellBuilder::new() - .store_dict_data(256, val_writer_ref_cell, value)? - .build()?; - - let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - - let dict = ContractLibraryDict { dict_boc, keys }; - Ok(dict) - } + pub keys: Vec, } #[async_trait] diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 999f611d..0bce0d6c 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -1,10 +1,6 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::atomic::{AtomicI32, Ordering}; +use std::collections::{HashSet, VecDeque}; use std::sync::Arc; -use std::time::Duration; -use futures::future::join_all; -use moka::future::Cache; use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; use tonlib_core::TonHash; @@ -12,47 +8,14 @@ use tonlib_core::TonHash; use super::{ContractLibraryDict, LibraryLoader}; use crate::contract::TonLibraryError; -const DEFAULT_LIBRARY_CACHE_CAPACITY: u64 = 300; -const DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); -const DEFAULT_INTERNAL_CACHE_CAPACITY: u64 = 300; -const DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); - -const DEFAULT_LIBRARY_CACHE_PARAMS: LibraryCacheParams = LibraryCacheParams { - lib_cache_capacity: DEFAULT_LIBRARY_CACHE_CAPACITY, - lib_cache_time_to_live: DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE, - internal_cache_capacity: DEFAULT_INTERNAL_CACHE_CAPACITY, - internal_cache_time_to_live: DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE, -}; - -#[derive(Clone, Copy)] -pub struct LibraryCacheParams { - lib_cache_capacity: u64, - lib_cache_time_to_live: Duration, - internal_cache_capacity: u64, - internal_cache_time_to_live: Duration, -} - #[derive(Clone)] pub struct LibraryProvider { - inner: Arc, -} - -struct Inner { loader: Arc, - cache_by_hash_seqno: Cache<(TonHash, i32), Option>, - cache_by_hash: Cache, - current_seqno: Arc, } impl LibraryProvider { - pub fn new( - loader: Arc, - cache_params: Option, - current_seqno: Arc, - ) -> Self { - Self { - inner: Arc::new(Inner::new(loader, cache_params, current_seqno)), - } + pub fn new(loader: Arc) -> Self { + Self { loader } } pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> @@ -85,145 +48,29 @@ impl LibraryProvider { Ok(lib_hashes) } - pub async fn get_libs_by_seqno<'a, I>( + pub async fn get_libs_dict<'a, I>( &self, cells: I, - seqno: i32, + seqno: Option, ) -> Result where I: IntoIterator, { - let lib_hashes = LibraryProvider::extract_library_hashes(cells)?; - let libs = self.inner.get_libs(lib_hashes.as_slice(), seqno).await?; - LibraryProvider::store_to_dict(lib_hashes.as_slice(), libs) - } + let refs = LibraryProvider::extract_library_hashes(cells)?; - pub async fn get_libs_latest<'a, I>( - &self, - cells: I, - ) -> Result - where - I: IntoIterator, - { - let seqno = self.inner.current_seqno.load(Ordering::Relaxed); - self.get_libs_by_seqno(cells, seqno).await - } + let libs = self.loader.load_libraries(refs.as_slice(), seqno).await?; + + let lib_hashmap = libs.into_iter().map(|l| (l.cell_hash(), l)).collect(); - fn store_to_dict( - library_hashes: &[TonHash], - lib_hashmap: HashMap, - ) -> Result { let lib_cell = CellBuilder::new() .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? .build()?; let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - let keys = library_hashes.to_vec(); - let dict = ContractLibraryDict { dict_boc, keys }; - Ok(dict) - } -} -impl Inner { - pub(crate) fn new( - loader: Arc, - cache_params: Option, - current_seqno: Arc, - ) -> Self { - let cache_params = cache_params.unwrap_or(DEFAULT_LIBRARY_CACHE_PARAMS); - - let library_cache = Cache::builder() - .max_capacity(cache_params.lib_cache_capacity) - .time_to_live(cache_params.lib_cache_time_to_live) - .build(); - let internal_cache = Cache::builder() - .max_capacity(cache_params.internal_cache_capacity) - .time_to_live(cache_params.internal_cache_time_to_live) - .build(); - Self { - loader, - cache_by_hash_seqno: library_cache, - cache_by_hash: internal_cache, - current_seqno, - } - } + let keys = refs.iter().map(|r| (*r).into()).collect(); - pub(crate) async fn get_libs( - &self, - lib_hashes: &[TonHash], - seqno: i32, - ) -> Result, TonLibraryError> -where { - let mut result_libs = HashMap::new(); - - let keys = lib_hashes - .iter() - .map(|h| (*h, seqno)) - .collect::>(); - let cached_libs_future = keys - .iter() - .map(|key| async move { (key.0, self.cache_by_hash_seqno.get(key).await) }); - let maybe_cached_libs = join_all(cached_libs_future).await; - - let mut hashes_to_load = vec![]; - for (hash, value) in maybe_cached_libs { - // outer option means whether library is in the cache or not - // inner option means whether library is available at certain seqno - match value { - Some(Some(lib)) => { - result_libs.insert(hash, lib); - } - Some(None) => {} - None => { - log::trace!("loading lib from BC: {:?}", hash); - hashes_to_load.push(hash); - } - } - } - - // load libs from blockchain - let mut blockchain_libs = self - .loader - .load_libraries(hashes_to_load.as_slice(), Some(seqno)) - .await?; - self.replace_by_existing_data(&mut blockchain_libs).await; - result_libs.extend(blockchain_libs.iter().map(|l| (l.cell_hash(), l.clone()))); - - self.insert_to_lib_cache(blockchain_libs, seqno).await?; - Ok(result_libs) - } - - async fn insert_to_lib_cache( - &self, - libs: Vec, - seqno: i32, - ) -> Result<(), TonLibraryError> { - let mut cache_insert_futures = Vec::with_capacity(libs.len()); - let mut internal_cache_insert_futures = Vec::with_capacity(libs.len()); - for lib in libs { - cache_insert_futures.push( - self.cache_by_hash_seqno - .insert((lib.cell_hash(), seqno), Some(lib.clone())), - ); - internal_cache_insert_futures.push(self.cache_by_hash.insert(lib.cell_hash(), lib)); - } - - join_all(cache_insert_futures).await; - join_all(internal_cache_insert_futures).await; - Ok(()) - } - - async fn replace_by_existing_data(&self, loaded_libs: &mut [ArcCell]) { - let loaded_libs_keys: Vec<_> = loaded_libs.iter().map(|l| l.cell_hash()).collect(); - let future_internals = loaded_libs_keys.iter().map(|k| self.cache_by_hash.get(k)); - - let internal_cached_libs = join_all(future_internals).await; - - // replace loaded libs from blockchain to ones from internal cache. - for i in 0..loaded_libs.len() { - if let Some(lib) = &internal_cached_libs[i] { - loaded_libs[i] = lib.clone() - } - } + let dict = ContractLibraryDict { dict_boc, keys }; + Ok(dict) } } diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index f6534490..f26acabd 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -102,7 +102,7 @@ impl TonContractState { let libs = self .factory .library_provider() - .get_libs_latest(&[code, data]) + .get_libs_dict(&[code, data], Some(self.account_state.block_id.seqno)) .await?; let run_result = unsafe { diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index ab46eea4..8aa26b56 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -114,7 +114,7 @@ async fn test_emulator_get_wallet_address() { #[tokio::test] async fn test_emulate_ston_router_v2() -> anyhow::Result<()> { common::init_logging(); - let client = common::new_archive_mainnet_client().await; + let client = common::new_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; @@ -244,7 +244,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { )?; let libs = factory .library_provider() - .get_libs_by_seqno(&[code_cell, data_cell], 47497716) + .get_libs_dict(&[code_cell, data_cell], None) .await?; let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( @@ -330,54 +330,3 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { Ok(()) } - -#[tokio::test] -async fn test_lib_cache_works() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - let factory = TonContractFactory::builder(&client) - .with_default_cache() - .build() - .await?; - - let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; - - let contract = factory.get_contract(&router_address); - let state = contract.get_state().await?; - - let code = state.get_account_state().code.clone(); - let data = state.get_account_state().data.clone(); - - let code_cell = BagOfCells::parse(&code)?.into_single_root()?; - let data_cell = BagOfCells::parse(&data)?.into_single_root()?; - - const MAX_ITER: usize = 100000; - - let mut all_libs = vec![]; - for i in 0..MAX_ITER { - let t = Instant::now(); - let libs = factory - .library_provider() - .get_libs_latest(&[code_cell.clone(), data_cell.clone()]) - .await?; - - if t.elapsed() > Duration::from_millis(10) { - log::info!("iteraion: {}DT {:?}", i, t.elapsed()); - } - - tokio::time::sleep(Duration::from_millis(10)).await; - - all_libs.push(libs) - } - - let bc_lib = &all_libs[0].dict_boc; - let cache_lib = &all_libs[1].dict_boc; - - let bc_cell = BagOfCells::parse(bc_lib)?.into_single_root()?; - - let cache_cell = BagOfCells::parse(cache_lib)?.into_single_root()?; - - assert_eq!(bc_cell, cache_cell); - - Ok(()) -} From 63eba992039ef93b97fe8c9c76cd1ebe11ef233d Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 14:21:31 +0400 Subject: [PATCH 10/18] Revert "retrying invokes on connection" This reverts commit 672bf3d12ed0fe3aeedc22ff89897d886a97fd3c. --- client/src/client/connection.rs | 36 +++++++-------------------------- client/src/client/types.rs | 7 ------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/client/src/client/connection.rs b/client/src/client/connection.rs index bbb47b05..6d6e76c0 100644 --- a/client/src/client/connection.rs +++ b/client/src/client/connection.rs @@ -7,10 +7,7 @@ use std::time::Instant; use async_trait::async_trait; use tokio::sync::{broadcast, oneshot, Mutex, Semaphore, SemaphorePermit}; -use tokio_retry::strategy::FixedInterval; -use tokio_retry::RetryIf; -use super::{retry_condition, RetryStrategy}; use crate::client::{ TonClientError, TonClientInterface, TonConnectionCallback, TonConnectionParams, TonNotificationReceiver, @@ -42,7 +39,6 @@ struct Inner { callback: Arc, _notification_receiver: TonNotificationReceiver, semaphore: Option, - retry_strategy: RetryStrategy, } pub struct TonConnection { @@ -90,8 +86,6 @@ impl TonConnection { None }; - let retry_strategy = params.retry_strategy.clone(); - let request_map = Mutex::new(HashMap::new()); let inner = Inner { tl_client: TlTonClient::new(tag.as_str()), @@ -101,7 +95,6 @@ impl TonConnection { callback, _notification_receiver: receiver, semaphore, - retry_strategy, }; let inner_arc = Arc::new(inner); let inner_weak: Weak = Arc::downgrade(&inner_arc); @@ -259,16 +252,15 @@ impl TonConnection { None }) } - async fn retrying_invoke( - &self, - function: &TonFunction, - ) -> Result<(TonConnection, TonResult), TonClientError> { - let fi = FixedInterval::from_millis(self.inner.retry_strategy.interval_ms); - let strategy = fi.take(self.inner.retry_strategy.max_retries); - RetryIf::spawn(strategy, || self.do_invoke(function), retry_condition).await +} + +#[async_trait] +impl TonClientInterface for TonConnection { + async fn get_connection(&self) -> Result { + Ok(self.clone()) } - async fn do_invoke( + async fn invoke_on_connection( &self, function: &TonFunction, ) -> Result<(TonConnection, TonResult), TonClientError> { @@ -310,20 +302,6 @@ impl TonConnection { } } -#[async_trait] -impl TonClientInterface for TonConnection { - async fn get_connection(&self) -> Result { - Ok(self.clone()) - } - - async fn invoke_on_connection( - &self, - function: &TonFunction, - ) -> Result<(TonConnection, TonResult), TonClientError> { - self.retrying_invoke(function).await - } -} - impl Clone for TonConnection { fn clone(&self) -> Self { let inner = self.inner.clone(); diff --git a/client/src/client/types.rs b/client/src/client/types.rs index 39ee72b8..b6f4d9c1 100644 --- a/client/src/client/types.rs +++ b/client/src/client/types.rs @@ -52,8 +52,6 @@ pub struct TonConnectionParams { pub concurrency_limit: usize, #[serde(default = "default_update_init_block")] pub update_init_block: bool, - #[serde(default = "default_retry_strategy")] - pub retry_strategy: RetryStrategy, } impl Default for TonConnectionParams { @@ -67,7 +65,6 @@ impl Default for TonConnectionParams { notification_queue_length: DEFAULT_NOTIFICATION_QUEUE_LENGTH, concurrency_limit: DEFAULT_CONNECTION_CONCURRENCY_LIMIT, update_init_block: DEFAULT_UPDATE_INIT_BLOCK, - retry_strategy: RetryStrategy::default(), } } } @@ -102,10 +99,6 @@ impl Default for RetryStrategy { } } -fn default_retry_strategy() -> RetryStrategy { - RetryStrategy::default() -} - lazy_static! { pub static ref DEFAULT_RETRY_STRATEGY: RetryStrategy = RetryStrategy::default(); } From 65a126ab8d5d2176ada0e65595627b3f3361c461 Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 14:26:27 +0400 Subject: [PATCH 11/18] Retrying invoke on connection --- client/src/client/connection.rs | 36 ++++++++++++++++++++++++++------- client/src/client/types.rs | 7 +++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/client/src/client/connection.rs b/client/src/client/connection.rs index 6d6e76c0..bbb47b05 100644 --- a/client/src/client/connection.rs +++ b/client/src/client/connection.rs @@ -7,7 +7,10 @@ use std::time::Instant; use async_trait::async_trait; use tokio::sync::{broadcast, oneshot, Mutex, Semaphore, SemaphorePermit}; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::RetryIf; +use super::{retry_condition, RetryStrategy}; use crate::client::{ TonClientError, TonClientInterface, TonConnectionCallback, TonConnectionParams, TonNotificationReceiver, @@ -39,6 +42,7 @@ struct Inner { callback: Arc, _notification_receiver: TonNotificationReceiver, semaphore: Option, + retry_strategy: RetryStrategy, } pub struct TonConnection { @@ -86,6 +90,8 @@ impl TonConnection { None }; + let retry_strategy = params.retry_strategy.clone(); + let request_map = Mutex::new(HashMap::new()); let inner = Inner { tl_client: TlTonClient::new(tag.as_str()), @@ -95,6 +101,7 @@ impl TonConnection { callback, _notification_receiver: receiver, semaphore, + retry_strategy, }; let inner_arc = Arc::new(inner); let inner_weak: Weak = Arc::downgrade(&inner_arc); @@ -252,15 +259,16 @@ impl TonConnection { None }) } -} - -#[async_trait] -impl TonClientInterface for TonConnection { - async fn get_connection(&self) -> Result { - Ok(self.clone()) + async fn retrying_invoke( + &self, + function: &TonFunction, + ) -> Result<(TonConnection, TonResult), TonClientError> { + let fi = FixedInterval::from_millis(self.inner.retry_strategy.interval_ms); + let strategy = fi.take(self.inner.retry_strategy.max_retries); + RetryIf::spawn(strategy, || self.do_invoke(function), retry_condition).await } - async fn invoke_on_connection( + async fn do_invoke( &self, function: &TonFunction, ) -> Result<(TonConnection, TonResult), TonClientError> { @@ -302,6 +310,20 @@ impl TonClientInterface for TonConnection { } } +#[async_trait] +impl TonClientInterface for TonConnection { + async fn get_connection(&self) -> Result { + Ok(self.clone()) + } + + async fn invoke_on_connection( + &self, + function: &TonFunction, + ) -> Result<(TonConnection, TonResult), TonClientError> { + self.retrying_invoke(function).await + } +} + impl Clone for TonConnection { fn clone(&self) -> Self { let inner = self.inner.clone(); diff --git a/client/src/client/types.rs b/client/src/client/types.rs index b6f4d9c1..39ee72b8 100644 --- a/client/src/client/types.rs +++ b/client/src/client/types.rs @@ -52,6 +52,8 @@ pub struct TonConnectionParams { pub concurrency_limit: usize, #[serde(default = "default_update_init_block")] pub update_init_block: bool, + #[serde(default = "default_retry_strategy")] + pub retry_strategy: RetryStrategy, } impl Default for TonConnectionParams { @@ -65,6 +67,7 @@ impl Default for TonConnectionParams { notification_queue_length: DEFAULT_NOTIFICATION_QUEUE_LENGTH, concurrency_limit: DEFAULT_CONNECTION_CONCURRENCY_LIMIT, update_init_block: DEFAULT_UPDATE_INIT_BLOCK, + retry_strategy: RetryStrategy::default(), } } } @@ -99,6 +102,10 @@ impl Default for RetryStrategy { } } +fn default_retry_strategy() -> RetryStrategy { + RetryStrategy::default() +} + lazy_static! { pub static ref DEFAULT_RETRY_STRATEGY: RetryStrategy = RetryStrategy::default(); } From 79535ea73d2fe4d1d227d05935f9d94cfd428c1a Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 15:26:40 +0400 Subject: [PATCH 12/18] Revert "Merge branch 'be-1879-3' into 'main'" This reverts commit 0a881796dfdb1ee3758b612e453eac30e4aedc64, reversing changes made to 63eba992039ef93b97fe8c9c76cd1ebe11ef233d. --- client/src/client/connection.rs | 36 +++++++-------------------------- client/src/client/types.rs | 7 ------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/client/src/client/connection.rs b/client/src/client/connection.rs index bbb47b05..6d6e76c0 100644 --- a/client/src/client/connection.rs +++ b/client/src/client/connection.rs @@ -7,10 +7,7 @@ use std::time::Instant; use async_trait::async_trait; use tokio::sync::{broadcast, oneshot, Mutex, Semaphore, SemaphorePermit}; -use tokio_retry::strategy::FixedInterval; -use tokio_retry::RetryIf; -use super::{retry_condition, RetryStrategy}; use crate::client::{ TonClientError, TonClientInterface, TonConnectionCallback, TonConnectionParams, TonNotificationReceiver, @@ -42,7 +39,6 @@ struct Inner { callback: Arc, _notification_receiver: TonNotificationReceiver, semaphore: Option, - retry_strategy: RetryStrategy, } pub struct TonConnection { @@ -90,8 +86,6 @@ impl TonConnection { None }; - let retry_strategy = params.retry_strategy.clone(); - let request_map = Mutex::new(HashMap::new()); let inner = Inner { tl_client: TlTonClient::new(tag.as_str()), @@ -101,7 +95,6 @@ impl TonConnection { callback, _notification_receiver: receiver, semaphore, - retry_strategy, }; let inner_arc = Arc::new(inner); let inner_weak: Weak = Arc::downgrade(&inner_arc); @@ -259,16 +252,15 @@ impl TonConnection { None }) } - async fn retrying_invoke( - &self, - function: &TonFunction, - ) -> Result<(TonConnection, TonResult), TonClientError> { - let fi = FixedInterval::from_millis(self.inner.retry_strategy.interval_ms); - let strategy = fi.take(self.inner.retry_strategy.max_retries); - RetryIf::spawn(strategy, || self.do_invoke(function), retry_condition).await +} + +#[async_trait] +impl TonClientInterface for TonConnection { + async fn get_connection(&self) -> Result { + Ok(self.clone()) } - async fn do_invoke( + async fn invoke_on_connection( &self, function: &TonFunction, ) -> Result<(TonConnection, TonResult), TonClientError> { @@ -310,20 +302,6 @@ impl TonConnection { } } -#[async_trait] -impl TonClientInterface for TonConnection { - async fn get_connection(&self) -> Result { - Ok(self.clone()) - } - - async fn invoke_on_connection( - &self, - function: &TonFunction, - ) -> Result<(TonConnection, TonResult), TonClientError> { - self.retrying_invoke(function).await - } -} - impl Clone for TonConnection { fn clone(&self) -> Self { let inner = self.inner.clone(); diff --git a/client/src/client/types.rs b/client/src/client/types.rs index 39ee72b8..b6f4d9c1 100644 --- a/client/src/client/types.rs +++ b/client/src/client/types.rs @@ -52,8 +52,6 @@ pub struct TonConnectionParams { pub concurrency_limit: usize, #[serde(default = "default_update_init_block")] pub update_init_block: bool, - #[serde(default = "default_retry_strategy")] - pub retry_strategy: RetryStrategy, } impl Default for TonConnectionParams { @@ -67,7 +65,6 @@ impl Default for TonConnectionParams { notification_queue_length: DEFAULT_NOTIFICATION_QUEUE_LENGTH, concurrency_limit: DEFAULT_CONNECTION_CONCURRENCY_LIMIT, update_init_block: DEFAULT_UPDATE_INIT_BLOCK, - retry_strategy: RetryStrategy::default(), } } } @@ -102,10 +99,6 @@ impl Default for RetryStrategy { } } -fn default_retry_strategy() -> RetryStrategy { - RetryStrategy::default() -} - lazy_static! { pub static ref DEFAULT_RETRY_STRATEGY: RetryStrategy = RetryStrategy::default(); } From b086d4075a51bc4fa59020c97b09d520d3dcd08c Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Mon, 6 Jan 2025 21:33:29 +0400 Subject: [PATCH 13/18] Reapply "Merge branch 'main' of ssh://git.stonfi.net:2222/ston-fi/rust/tonlib-rs into BE-1879" This reverts commit f454e75954c53c4e67ff5206e2331fa9dd7b78ea. --- client/src/contract/factory.rs | 3 + client/src/contract/factory/builder.rs | 12 +- client/src/contract/factory/cache.rs | 19 +- client/src/contract/factory/library_loader.rs | 24 ++- .../src/contract/factory/library_provider.rs | 177 ++++++++++++++++-- client/src/contract/state.rs | 2 +- client/tests/contract_emulator_test.rs | 55 +++++- 7 files changed, 268 insertions(+), 24 deletions(-) diff --git a/client/src/contract/factory.rs b/client/src/contract/factory.rs index bbbf346e..61ff18b1 100644 --- a/client/src/contract/factory.rs +++ b/client/src/contract/factory.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -44,6 +45,7 @@ impl TonContractFactory { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, + current_seqno: Arc, ) -> Result { let cache = if with_cache { let cache = ContractFactoryCache::new( @@ -53,6 +55,7 @@ impl TonContractFactory { txid_cache_capacity, txid_cache_time_to_live, presync_blocks, + current_seqno, ) .await?; Some(cache) diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index dff2cbaf..9819d856 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; @@ -14,6 +15,7 @@ pub struct TonContractFactoryBuilder { txid_cache_time_to_live: Duration, presync_blocks: i32, library_provider: LibraryProvider, + current_seqno: Arc, } impl TonContractFactoryBuilder { @@ -26,17 +28,19 @@ impl TonContractFactoryBuilder { const DEFAULT_PRESYNC_BLOCKS: i32 = 50; pub(crate) fn new(client: &TonClient) -> Self { + let current_seqno_counter: Arc = Arc::new(0.into()); let loader = BlockchainLibraryLoader::new(client); - let library_provider = LibraryProvider::new(loader); + let library_provider = LibraryProvider::new(loader, None, current_seqno_counter.clone()); TonContractFactoryBuilder { client: client.clone(), - with_cache: false, + with_cache: true, account_state_cache_capacity: 0, account_state_cache_time_to_live: Duration::default(), txid_cache_capacity: 0, txid_cache_time_to_live: Duration::default(), presync_blocks: Self::DEFAULT_PRESYNC_BLOCKS, library_provider, + current_seqno: current_seqno_counter, } } @@ -79,11 +83,13 @@ impl TonContractFactoryBuilder { self.txid_cache_time_to_live, self.presync_blocks, self.library_provider.clone(), + self.current_seqno.clone(), ) .await } pub fn with_library_loader(mut self, library_loader: &Arc) -> Self { - let library_provider = LibraryProvider::new(library_loader.clone()); + let library_provider = + LibraryProvider::new(library_loader.clone(), None, self.current_seqno.clone()); self.library_provider = library_provider; self } diff --git a/client/src/contract/factory/cache.rs b/client/src/contract/factory/cache.rs index 16103101..bc8e62ba 100644 --- a/client/src/contract/factory/cache.rs +++ b/client/src/contract/factory/cache.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -33,6 +33,7 @@ impl ContractFactoryCache { txid_cache_capacity: u64, txid_state_cache_time_to_live: Duration, presync_blocks: i32, + current_seqno: Arc, ) -> Result { let inner = Inner { client: client.clone(), @@ -49,6 +50,7 @@ impl ContractFactoryCache { account_state_cache_counters: ContractFactoryCacheCounters::default(), tx_id_cache_counters: ContractFactoryCacheCounters::default(), + current_seqno, }; let arc_inner = Arc::new(inner); @@ -137,6 +139,7 @@ impl ContractFactoryCache { match masterchain_info_result { Ok((_, info)) => { let first_block_seqno = info.last.seqno - inner.presync_blocks; + let block_stream = BlockStream::new(client, first_block_seqno); break block_stream; } @@ -161,10 +164,16 @@ impl ContractFactoryCache { if weak_inner.upgrade().is_none() { break; } - let block_result = block_stream.next().await; let block = match block_result { - Ok(block) => block, + Ok(block) => { + if let Some(inner) = weak_inner.upgrade() { + inner + .current_seqno + .store(block.master_shard.seqno, Ordering::Relaxed) + }; + block + } Err(e) => { log::warn!( "[ContractFactoryCache] Could not retrieve next block: {:?}", @@ -177,6 +186,9 @@ impl ContractFactoryCache { loop { if let Some(inner) = weak_inner.upgrade() { + inner + .current_seqno + .store(block.master_shard.seqno, Ordering::Relaxed); let process_result = inner.process_next_block(&block).await; match process_result { Ok(_) => break, @@ -227,6 +239,7 @@ struct Inner { presync_blocks: i32, tx_id_cache_counters: ContractFactoryCacheCounters, account_state_cache_counters: ContractFactoryCacheCounters, + current_seqno: Arc, } impl Inner { diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index 626414f0..df89a307 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -1,17 +1,35 @@ +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use tonlib_core::cell::{ArcCell, BagOfCells}; +use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; +use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder, TonCellError}; use tonlib_core::TonHash; use crate::client::{TonClient, TonClientInterface}; use crate::contract::TonLibraryError; use crate::tl::TonLibraryId; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct ContractLibraryDict { pub dict_boc: Vec, - pub keys: Vec, + pub keys: Vec, +} + +impl TryFrom> for ContractLibraryDict { + type Error = TonCellError; + + fn try_from(value: HashMap) -> Result { + let keys = value.keys().copied().collect(); + let lib_cell = CellBuilder::new() + .store_dict_data(256, val_writer_ref_cell, value)? + .build()?; + + let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; + + let dict = ContractLibraryDict { dict_boc, keys }; + Ok(dict) + } } #[async_trait] diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 0bce0d6c..999f611d 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -1,6 +1,10 @@ -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; +use std::time::Duration; +use futures::future::join_all; +use moka::future::Cache; use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; use tonlib_core::TonHash; @@ -8,14 +12,47 @@ use tonlib_core::TonHash; use super::{ContractLibraryDict, LibraryLoader}; use crate::contract::TonLibraryError; +const DEFAULT_LIBRARY_CACHE_CAPACITY: u64 = 300; +const DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); +const DEFAULT_INTERNAL_CACHE_CAPACITY: u64 = 300; +const DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); + +const DEFAULT_LIBRARY_CACHE_PARAMS: LibraryCacheParams = LibraryCacheParams { + lib_cache_capacity: DEFAULT_LIBRARY_CACHE_CAPACITY, + lib_cache_time_to_live: DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE, + internal_cache_capacity: DEFAULT_INTERNAL_CACHE_CAPACITY, + internal_cache_time_to_live: DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE, +}; + +#[derive(Clone, Copy)] +pub struct LibraryCacheParams { + lib_cache_capacity: u64, + lib_cache_time_to_live: Duration, + internal_cache_capacity: u64, + internal_cache_time_to_live: Duration, +} + #[derive(Clone)] pub struct LibraryProvider { + inner: Arc, +} + +struct Inner { loader: Arc, + cache_by_hash_seqno: Cache<(TonHash, i32), Option>, + cache_by_hash: Cache, + current_seqno: Arc, } impl LibraryProvider { - pub fn new(loader: Arc) -> Self { - Self { loader } + pub fn new( + loader: Arc, + cache_params: Option, + current_seqno: Arc, + ) -> Self { + Self { + inner: Arc::new(Inner::new(loader, cache_params, current_seqno)), + } } pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> @@ -48,29 +85,145 @@ impl LibraryProvider { Ok(lib_hashes) } - pub async fn get_libs_dict<'a, I>( + pub async fn get_libs_by_seqno<'a, I>( &self, cells: I, - seqno: Option, + seqno: i32, ) -> Result where I: IntoIterator, { - let refs = LibraryProvider::extract_library_hashes(cells)?; - - let libs = self.loader.load_libraries(refs.as_slice(), seqno).await?; + let lib_hashes = LibraryProvider::extract_library_hashes(cells)?; + let libs = self.inner.get_libs(lib_hashes.as_slice(), seqno).await?; + LibraryProvider::store_to_dict(lib_hashes.as_slice(), libs) + } - let lib_hashmap = libs.into_iter().map(|l| (l.cell_hash(), l)).collect(); + pub async fn get_libs_latest<'a, I>( + &self, + cells: I, + ) -> Result + where + I: IntoIterator, + { + let seqno = self.inner.current_seqno.load(Ordering::Relaxed); + self.get_libs_by_seqno(cells, seqno).await + } + fn store_to_dict( + library_hashes: &[TonHash], + lib_hashmap: HashMap, + ) -> Result { let lib_cell = CellBuilder::new() .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? .build()?; let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - - let keys = refs.iter().map(|r| (*r).into()).collect(); - + let keys = library_hashes.to_vec(); let dict = ContractLibraryDict { dict_boc, keys }; Ok(dict) } } + +impl Inner { + pub(crate) fn new( + loader: Arc, + cache_params: Option, + current_seqno: Arc, + ) -> Self { + let cache_params = cache_params.unwrap_or(DEFAULT_LIBRARY_CACHE_PARAMS); + + let library_cache = Cache::builder() + .max_capacity(cache_params.lib_cache_capacity) + .time_to_live(cache_params.lib_cache_time_to_live) + .build(); + let internal_cache = Cache::builder() + .max_capacity(cache_params.internal_cache_capacity) + .time_to_live(cache_params.internal_cache_time_to_live) + .build(); + Self { + loader, + cache_by_hash_seqno: library_cache, + cache_by_hash: internal_cache, + current_seqno, + } + } + + pub(crate) async fn get_libs( + &self, + lib_hashes: &[TonHash], + seqno: i32, + ) -> Result, TonLibraryError> +where { + let mut result_libs = HashMap::new(); + + let keys = lib_hashes + .iter() + .map(|h| (*h, seqno)) + .collect::>(); + let cached_libs_future = keys + .iter() + .map(|key| async move { (key.0, self.cache_by_hash_seqno.get(key).await) }); + let maybe_cached_libs = join_all(cached_libs_future).await; + + let mut hashes_to_load = vec![]; + for (hash, value) in maybe_cached_libs { + // outer option means whether library is in the cache or not + // inner option means whether library is available at certain seqno + match value { + Some(Some(lib)) => { + result_libs.insert(hash, lib); + } + Some(None) => {} + None => { + log::trace!("loading lib from BC: {:?}", hash); + hashes_to_load.push(hash); + } + } + } + + // load libs from blockchain + let mut blockchain_libs = self + .loader + .load_libraries(hashes_to_load.as_slice(), Some(seqno)) + .await?; + self.replace_by_existing_data(&mut blockchain_libs).await; + result_libs.extend(blockchain_libs.iter().map(|l| (l.cell_hash(), l.clone()))); + + self.insert_to_lib_cache(blockchain_libs, seqno).await?; + Ok(result_libs) + } + + async fn insert_to_lib_cache( + &self, + libs: Vec, + seqno: i32, + ) -> Result<(), TonLibraryError> { + let mut cache_insert_futures = Vec::with_capacity(libs.len()); + let mut internal_cache_insert_futures = Vec::with_capacity(libs.len()); + for lib in libs { + cache_insert_futures.push( + self.cache_by_hash_seqno + .insert((lib.cell_hash(), seqno), Some(lib.clone())), + ); + internal_cache_insert_futures.push(self.cache_by_hash.insert(lib.cell_hash(), lib)); + } + + join_all(cache_insert_futures).await; + join_all(internal_cache_insert_futures).await; + Ok(()) + } + + async fn replace_by_existing_data(&self, loaded_libs: &mut [ArcCell]) { + let loaded_libs_keys: Vec<_> = loaded_libs.iter().map(|l| l.cell_hash()).collect(); + let future_internals = loaded_libs_keys.iter().map(|k| self.cache_by_hash.get(k)); + + let internal_cached_libs = join_all(future_internals).await; + + // replace loaded libs from blockchain to ones from internal cache. + for i in 0..loaded_libs.len() { + if let Some(lib) = &internal_cached_libs[i] { + loaded_libs[i] = lib.clone() + } + } + } +} diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index f26acabd..f6534490 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -102,7 +102,7 @@ impl TonContractState { let libs = self .factory .library_provider() - .get_libs_dict(&[code, data], Some(self.account_state.block_id.seqno)) + .get_libs_latest(&[code, data]) .await?; let run_result = unsafe { diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 8aa26b56..ab46eea4 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -114,7 +114,7 @@ async fn test_emulator_get_wallet_address() { #[tokio::test] async fn test_emulate_ston_router_v2() -> anyhow::Result<()> { common::init_logging(); - let client = common::new_mainnet_client().await; + let client = common::new_archive_mainnet_client().await; let factory = TonContractFactory::builder(&client).build().await?; let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; @@ -244,7 +244,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { )?; let libs = factory .library_provider() - .get_libs_dict(&[code_cell, data_cell], None) + .get_libs_by_seqno(&[code_cell, data_cell], 47497716) .await?; let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( @@ -330,3 +330,54 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_lib_cache_works() -> anyhow::Result<()> { + common::init_logging(); + let client = common::new_mainnet_client().await; + let factory = TonContractFactory::builder(&client) + .with_default_cache() + .build() + .await?; + + let router_address = "EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB".parse()?; + + let contract = factory.get_contract(&router_address); + let state = contract.get_state().await?; + + let code = state.get_account_state().code.clone(); + let data = state.get_account_state().data.clone(); + + let code_cell = BagOfCells::parse(&code)?.into_single_root()?; + let data_cell = BagOfCells::parse(&data)?.into_single_root()?; + + const MAX_ITER: usize = 100000; + + let mut all_libs = vec![]; + for i in 0..MAX_ITER { + let t = Instant::now(); + let libs = factory + .library_provider() + .get_libs_latest(&[code_cell.clone(), data_cell.clone()]) + .await?; + + if t.elapsed() > Duration::from_millis(10) { + log::info!("iteraion: {}DT {:?}", i, t.elapsed()); + } + + tokio::time::sleep(Duration::from_millis(10)).await; + + all_libs.push(libs) + } + + let bc_lib = &all_libs[0].dict_boc; + let cache_lib = &all_libs[1].dict_boc; + + let bc_cell = BagOfCells::parse(bc_lib)?.into_single_root()?; + + let cache_cell = BagOfCells::parse(cache_lib)?.into_single_root()?; + + assert_eq!(bc_cell, cache_cell); + + Ok(()) +} From b591f6959bd8347cf409d0da848778653cc44e02 Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Tue, 7 Jan 2025 14:19:45 +0400 Subject: [PATCH 14/18] impl display for TonHash --- client/src/contract/factory/library_loader.rs | 20 +------------------ core/src/types.rs | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs index df89a307..3e4f60d5 100644 --- a/client/src/contract/factory/library_loader.rs +++ b/client/src/contract/factory/library_loader.rs @@ -1,9 +1,7 @@ -use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; -use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder, TonCellError}; +use tonlib_core::cell::{ArcCell, BagOfCells}; use tonlib_core::TonHash; use crate::client::{TonClient, TonClientInterface}; @@ -16,22 +14,6 @@ pub struct ContractLibraryDict { pub keys: Vec, } -impl TryFrom> for ContractLibraryDict { - type Error = TonCellError; - - fn try_from(value: HashMap) -> Result { - let keys = value.keys().copied().collect(); - let lib_cell = CellBuilder::new() - .store_dict_data(256, val_writer_ref_cell, value)? - .build()?; - - let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - - let dict = ContractLibraryDict { dict_boc, keys }; - Ok(dict) - } -} - #[async_trait] pub trait LibraryLoader: Send + Sync { async fn load_libraries( diff --git a/core/src/types.rs b/core/src/types.rs index 99e8927f..7dcf1c95 100644 --- a/core/src/types.rs +++ b/core/src/types.rs @@ -2,6 +2,8 @@ mod address; mod error; mod tx_id; +use std::fmt; + pub use address::*; use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; @@ -19,7 +21,7 @@ pub const DEFAULT_CELL_HASH: TonHash = TonHash([ 93, 197, 184, 126, 65, 11, 120, 99, 10, 9, 207, 199, ]); -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct TonHash([u8; TON_HASH_LEN]); impl TonHash { @@ -113,3 +115,19 @@ impl TryFrom> for TonHash { Self::try_from(value.as_slice()) } } + +impl fmt::Debug for TonHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Delegate to Display for Debug formatting + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for TonHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in &self.0 { + write!(f, "{:02x}", byte)?; + } + Ok(()) + } +} From 925ccd7c48265495e9d4e83a79102c2aaf1190f4 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Tue, 7 Jan 2025 13:24:21 +0000 Subject: [PATCH 15/18] Impl #BE-1839: Boxed long errors, updated builder methods --- client/src/contract/error.rs | 8 ++++---- client/src/contract/factory/builder.rs | 12 ++++++------ client/src/contract/state.rs | 14 ++++++++------ client/src/emulator/c7_register.rs | 6 +++--- client/src/emulator/tvm_emulator.rs | 8 ++++---- client/tests/contract_emulator_test.rs | 14 +++++++------- client/tests/error_test.rs | 6 +++--- client/tests/tvm_emulator_test.rs | 11 ++++++----- 8 files changed, 41 insertions(+), 38 deletions(-) diff --git a/client/src/contract/error.rs b/client/src/contract/error.rs index 88bb47b5..c4b81b0a 100644 --- a/client/src/contract/error.rs +++ b/client/src/contract/error.rs @@ -79,7 +79,7 @@ pub enum TonContractError { TvmStackParseError { method: TonMethodId, address: TonAddress, - error: StackParseError, + error: Box, }, #[error( @@ -89,9 +89,9 @@ pub enum TonContractError { TvmRunError { method: TonMethodId, address: TonAddress, - vm_log: Option, + vm_log: Box>, exit_code: i32, - stack: Vec, + stack: Box>, missing_library: Option, gas_used: i64, }, @@ -146,7 +146,7 @@ impl MapStackError for Result { self.map_err(|e| TonContractError::TvmStackParseError { method: TonMethodId::Name(method.into()), address: address.clone(), - error: e, + error: e.into(), }) } } diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index 9819d856..c38ac377 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -45,12 +45,12 @@ impl TonContractFactoryBuilder { } pub fn with_cache( - mut self, + &mut self, txid_cache_capacity: u64, txid_cache_time_to_live: Duration, account_state_cache_capacity: u64, account_state_cache_time_to_live: Duration, - ) -> Self { + ) -> &mut Self { self.with_cache = true; self.txid_cache_capacity = txid_cache_capacity; self.txid_cache_time_to_live = txid_cache_time_to_live; @@ -59,7 +59,7 @@ impl TonContractFactoryBuilder { self } - pub fn with_default_cache(mut self) -> Self { + pub fn with_default_cache(&mut self) -> &mut Self { self.with_cache = true; self.account_state_cache_capacity = Self::DEFAULT_ACCOUNT_STATE_CACHE_CAPACITY; self.account_state_cache_time_to_live = Self::DEFAULT_ACCOUNT_STATE_CACHE_TTL; @@ -68,7 +68,7 @@ impl TonContractFactoryBuilder { self } - pub fn with_presync_blocks(mut self, presync_blocks: i32) -> Self { + pub fn with_presync_blocks(&mut self, presync_blocks: i32) -> &mut Self { self.presync_blocks = presync_blocks; self } @@ -87,14 +87,14 @@ impl TonContractFactoryBuilder { ) .await } - pub fn with_library_loader(mut self, library_loader: &Arc) -> Self { + pub fn with_library_loader(&mut self, library_loader: &Arc) -> &mut Self { let library_provider = LibraryProvider::new(library_loader.clone(), None, self.current_seqno.clone()); self.library_provider = library_provider; self } - pub fn with_library_provider(mut self, library_provider: &LibraryProvider) -> Self { + pub fn with_library_provider(&mut self, library_provider: &LibraryProvider) -> &mut Self { self.library_provider = library_provider.clone(); self } diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index f6534490..9c6c7870 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -118,7 +118,8 @@ impl TonContractState { tokio::task::spawn_blocking(move || { let code = state.code.as_slice(); let data = state.data.as_slice(); - let mut emulator = TvmEmulator::new(code, data)? + let mut emulator = TvmEmulator::new(code, data)?; + emulator .with_c7(&c7)? .with_libraries(libs.dict_boc.as_slice())?; let run_result = emulator.run_get_method(&static_method_id, static_stack); @@ -148,7 +149,8 @@ impl TonContractState { let run_result = tokio::task::spawn_blocking(move || { let code = state.code.as_slice(); let data = state.data.as_slice(); - let mut emulator = TvmEmulator::new(code, data)?.with_c7(&c7)?; + let mut emulator = TvmEmulator::new(code, data)?; + emulator.with_c7(&c7)?; emulator.send_internal_message(message, amount) }) .await @@ -193,7 +195,7 @@ impl TonContractState { .map_err(|e| TonContractError::TvmStackParseError { method: method.into(), address: self.address().clone(), - error: e, + error: Box::new(e), })?; let run_result = state @@ -213,7 +215,7 @@ impl TonContractState { .map_err(|e| TonContractError::TvmStackParseError { method: method.into(), address: self.address().clone(), - error: e, + error: Box::new(e), })?; let result = TvmSuccess { vm_log: None, @@ -236,9 +238,9 @@ impl TonContractState { method: method.clone(), address: address.clone(), gas_used: run_result.gas_used.into(), - stack: run_result.stack, + stack: Box::new(run_result.stack), exit_code: run_result.vm_exit_code, - vm_log: run_result.vm_log, + vm_log: Box::new(run_result.vm_log), missing_library: run_result.missing_library, }) } else { diff --git a/client/src/emulator/c7_register.rs b/client/src/emulator/c7_register.rs index 0578525c..c5aeee73 100644 --- a/client/src/emulator/c7_register.rs +++ b/client/src/emulator/c7_register.rs @@ -31,17 +31,17 @@ impl TvmEmulatorC7 { Ok(c7) } - pub fn with_balance(mut self, balance: u64) -> Self { + pub fn with_balance(&mut self, balance: u64) -> &mut Self { self.balance = balance; self } - pub fn with_seed(mut self, seed: TonHash) -> Self { + pub fn with_seed(&mut self, seed: TonHash) -> &mut Self { self.seed = seed; self } - pub fn with_unix_time(mut self, unix_time: u64) -> Self { + pub fn with_unix_time(&mut self, unix_time: u64) -> &mut Self { self.unix_time = unix_time; self } diff --git a/client/src/emulator/tvm_emulator.rs b/client/src/emulator/tvm_emulator.rs index 1d8e010b..96d68eda 100644 --- a/client/src/emulator/tvm_emulator.rs +++ b/client/src/emulator/tvm_emulator.rs @@ -22,7 +22,7 @@ impl TvmEmulator { Ok(ton_contract_emulator) } - pub fn with_c7(mut self, c7: &TvmEmulatorC7) -> Result { + pub fn with_c7(&mut self, c7: &TvmEmulatorC7) -> Result<&mut Self, TvmEmulatorError> { let addr_str = c7.address.to_hex(); let hex_str = c7.seed.to_hex(); let seed = hex_str.as_bytes(); @@ -42,7 +42,7 @@ impl TvmEmulator { )) } - pub fn with_debug_enabled(mut self) -> Result { + pub fn with_debug_enabled(&mut self) -> Result<&mut Self, TvmEmulatorError> { if self.emulator.set_debug_enabled(true) { return Ok(self); } @@ -51,7 +51,7 @@ impl TvmEmulator { )) } - pub fn with_gas_limit(mut self, gas_limit: u64) -> Result { + pub fn with_gas_limit(&mut self, gas_limit: u64) -> Result<&mut Self, TvmEmulatorError> { if self.emulator.set_gas_limit(gas_limit) { return Ok(self); } @@ -60,7 +60,7 @@ impl TvmEmulator { )) } - pub fn with_libraries(mut self, libraries: &[u8]) -> Result { + pub fn with_libraries(&mut self, libraries: &[u8]) -> Result<&mut Self, TvmEmulatorError> { if libraries.is_empty() { return Ok(self); } diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index ab46eea4..22a80d46 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -42,14 +42,14 @@ async fn emulate_get_jetton_data( address: &TonAddress, ) -> anyhow::Result { let config = factory.get_config_cell_serial().await?; - let c7 = assert_ok!(TvmEmulatorC7::new(address.clone(), Vec::from(config))) - .with_balance(account_state.balance as u64); + let mut c7 = assert_ok!(TvmEmulatorC7::new(address.clone(), Vec::from(config))); + c7.with_balance(account_state.balance as u64); let mut emulator = assert_ok!(TvmEmulator::new( account_state.code.as_slice(), account_state.data.as_slice() - )) - .with_c7(&c7)?; + )); + emulator.with_c7(&c7)?; let result = assert_ok!(emulator.run_get_method(&"get_jetton_data".into(), &[])); assert!(result.exit_success()); @@ -284,7 +284,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { let data = data.as_slice(); let t_creation = Instant::now(); - let emulator = TvmEmulator::new(code, data).unwrap(); + let mut emulator = TvmEmulator::new(code, data).unwrap(); let creation_time = t_creation.elapsed(); let t_c7 = Instant::now(); @@ -292,7 +292,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { let c7_time = t_c7.elapsed(); let t_lib = Instant::now(); - let mut e = e.with_libraries(libs.as_slice()).unwrap(); + let e = e.with_libraries(libs.as_slice()).unwrap(); let lib_time = t_lib.elapsed(); let running_time = Instant::now(); @@ -351,7 +351,7 @@ async fn test_lib_cache_works() -> anyhow::Result<()> { let code_cell = BagOfCells::parse(&code)?.into_single_root()?; let data_cell = BagOfCells::parse(&data)?.into_single_root()?; - const MAX_ITER: usize = 100000; + const MAX_ITER: usize = 1000; let mut all_libs = vec![]; for i in 0..MAX_ITER { diff --git a/client/tests/error_test.rs b/client/tests/error_test.rs index d1e1a91f..39be32db 100644 --- a/client/tests/error_test.rs +++ b/client/tests/error_test.rs @@ -203,13 +203,13 @@ fn test_ton_contract_error_output() { TonContractError::TvmRunError { method: "some_get_method".into(), gas_used: 300, - stack: vec![ + stack: Box::new(vec![ TvmStackEntry::Slice(assert_ok!(CellSlice::full_cell(cell.clone()))), TvmStackEntry::Cell(Arc::new(cell)), TvmStackEntry::Int257(BigInt::from(1234566789)), - ], + ]), exit_code: -123, - vm_log: None, + vm_log: Box::new(None), missing_library: None, address: TonAddress::null(), } diff --git a/client/tests/tvm_emulator_test.rs b/client/tests/tvm_emulator_test.rs index dc9c2466..1c244793 100644 --- a/client/tests/tvm_emulator_test.rs +++ b/client/tests/tvm_emulator_test.rs @@ -159,8 +159,8 @@ async fn test_emulator_bigint_multiply() -> anyhow::Result<()> { fn bigint_multiplier(val1: &BigInt, val2: &BigInt) -> anyhow::Result<()> { let expected = val1 * val2; log::info!("Testing: {} = {} * {}", expected, val1, val2); - let mut emulator = - TvmEmulator::new(&TEST_CONTRACT_CODE, &TEST_CONTRACT_DATA)?.with_debug_enabled()?; + let mut emulator = TvmEmulator::new(&TEST_CONTRACT_CODE, &TEST_CONTRACT_DATA)?; + emulator.with_debug_enabled()?; let stack = vec![val1.clone().into(), val2.clone().into()]; let emulator_result = assert_ok!(emulator.run_get_method(&"get_val".into(), stack.as_slice())); assert_eq!(emulator_result.vm_exit_code, 0); @@ -184,8 +184,8 @@ async fn test_emulator_i64_multiply() -> anyhow::Result<()> { fn i64_multiplier(val1: i64, val2: i64) -> anyhow::Result<()> { let expected = BigInt::from(val1) * BigInt::from(val2); log::info!("Testing: {} = {} * {}", expected, val1, val2); - let mut emulator = - TvmEmulator::new(&TEST_CONTRACT_CODE, &TEST_CONTRACT_DATA)?.with_debug_enabled()?; + let mut emulator = TvmEmulator::new(&TEST_CONTRACT_CODE, &TEST_CONTRACT_DATA)?; + emulator.with_debug_enabled()?; let stack = vec![val1.into(), val2.into()]; let emulator_result = assert_ok!(emulator.run_get_method(&"get_val".into(), stack.as_slice())); assert_eq!(emulator_result.vm_exit_code, 0); @@ -318,7 +318,8 @@ fn emulate_get_wallet_address( config_data: &[u8], ) -> anyhow::Result { let tvm_emulator_c7 = TvmEmulatorC7::new(self_address.clone(), config_data.to_vec())?; - let mut emulator = TvmEmulator::new(code, data)?.with_c7(&tvm_emulator_c7)?; + let mut emulator = TvmEmulator::new(code, data)?; + emulator.with_c7(&tvm_emulator_c7)?; let stack: Vec = vec![assert_ok!(owner_address.try_into())]; let emulator_result = From 07f1890c3aafddc1b81ca0ee0c31c95f8eb9c139 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 9 Jan 2025 11:22:45 +0000 Subject: [PATCH 16/18] Impl: #BE-1893: Library provider trait and implementation of BlockChainLibraryProvider --- client/src/contract/error.rs | 3 + client/src/contract/factory.rs | 13 +- .../factory/blockchain_library_provider.rs | 156 ++++++++++++ client/src/contract/factory/builder.rs | 19 +- client/src/contract/factory/library_helper.rs | 81 ++++++ client/src/contract/factory/library_loader.rs | 77 ------ .../src/contract/factory/library_provider.rs | 233 +----------------- client/src/contract/state.rs | 2 +- client/tests/contract_emulator_test.rs | 4 +- client/tests/library_test.rs | 44 +--- 10 files changed, 279 insertions(+), 353 deletions(-) create mode 100644 client/src/contract/factory/blockchain_library_provider.rs create mode 100644 client/src/contract/factory/library_helper.rs delete mode 100644 client/src/contract/factory/library_loader.rs diff --git a/client/src/contract/error.rs b/client/src/contract/error.rs index c4b81b0a..0a202fef 100644 --- a/client/src/contract/error.rs +++ b/client/src/contract/error.rs @@ -181,4 +181,7 @@ pub enum TonLibraryError { #[error("Expected exactly one library, but got multiple")] MultipleLibrariesReturned, + + #[error("Getting library by mc_seqno is not supported")] + SeqnoNotSupported, } diff --git a/client/src/contract/factory.rs b/client/src/contract/factory.rs index 61ff18b1..e98c1f57 100644 --- a/client/src/contract/factory.rs +++ b/client/src/contract/factory.rs @@ -2,9 +2,10 @@ use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; +pub use blockchain_library_provider::*; pub use builder::*; pub use cache::*; -pub use library_loader::*; +pub use library_helper::*; pub use library_provider::*; use tokio::sync::OnceCell; use tonlib_core::TonAddress; @@ -13,11 +14,11 @@ use crate::client::{TonClient, TonClientError, TonClientInterface}; use crate::contract::{LoadedSmcState, TonContract, TonContractError, TonContractState}; use crate::tl::{ConfigInfo, InternalTransactionId, RawFullAccountState}; +mod blockchain_library_provider; mod builder; mod cache; -mod library_loader; +mod library_helper; mod library_provider; - #[derive(Clone)] pub struct TonContractFactory { inner: Arc, @@ -26,7 +27,7 @@ pub struct TonContractFactory { struct Inner { client: TonClient, config_info: OnceCell, - library_provider: LibraryProvider, + library_provider: Arc, cache: Option, } @@ -44,7 +45,7 @@ impl TonContractFactory { txid_cache_capacity: u64, txid_cache_time_to_live: Duration, presync_blocks: i32, - library_provider: LibraryProvider, + library_provider: Arc, current_seqno: Arc, ) -> Result { let cache = if with_cache { @@ -88,7 +89,7 @@ impl TonContractFactory { Ok(c.config.bytes.as_slice()) } - pub fn library_provider(&self) -> LibraryProvider { + pub fn library_provider(&self) -> Arc { self.inner.library_provider.clone() } diff --git a/client/src/contract/factory/blockchain_library_provider.rs b/client/src/contract/factory/blockchain_library_provider.rs new file mode 100644 index 00000000..de563862 --- /dev/null +++ b/client/src/contract/factory/blockchain_library_provider.rs @@ -0,0 +1,156 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use futures::future::join_all; +use moka::future::Cache; +use tonlib_core::cell::{ArcCell, BagOfCells}; +use tonlib_core::TonHash; + +use super::{ContractLibraryDict, LibraryHelper, LibraryProvider}; +use crate::client::{TonClient, TonClientInterface}; +use crate::contract::TonLibraryError; +use crate::tl::TonLibraryId; + +#[derive(Clone, Copy)] +pub struct LibraryCacheParams { + capacity: u64, + time_to_live: Duration, +} + +impl Default for LibraryCacheParams { + fn default() -> Self { + const DEFAULT_LIBRARY_CACHE_CAPACITY: u64 = 300; + const DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); + Self { + capacity: DEFAULT_LIBRARY_CACHE_CAPACITY, + time_to_live: DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE, + } + } +} + +#[derive(Clone)] +pub struct BlockchainLibraryProvider { + inner: Arc, +} + +struct Inner { + client: TonClient, + cache: Cache, +} + +#[async_trait] +impl LibraryProvider for BlockchainLibraryProvider { + async fn get_libs( + &self, + cells: &[ArcCell], + mc_seqno: Option, + ) -> Result { + if mc_seqno.is_some() { + return Err(TonLibraryError::SeqnoNotSupported); + } + + let lib_hashes = LibraryHelper::extract_library_hashes(cells)?; + let libs = self.inner.get_libs(lib_hashes.as_slice()).await?; + LibraryHelper::store_to_dict(lib_hashes.as_slice(), libs) + } +} + +impl BlockchainLibraryProvider { + const MAX_LIBS_REQUESTED: usize = 255; + + pub fn new(client: &TonClient, cache_params: Option) -> Self { + Self { + inner: Arc::new(Inner::new(client, cache_params)), + } + } + + pub async fn load_libraries( + &self, + hashes: &[TonHash], + ) -> Result, TonLibraryError> { + let mut results = Vec::new(); + + // If hashes exceed MAX_LIBS_REQUESTED, split them into chunks + for chunk in hashes.chunks(Self::MAX_LIBS_REQUESTED) { + let mut partial_result = self.inner.load_libraries_impl(chunk).await?; + results.append(&mut partial_result); + } + Ok(results) + } +} + +impl Inner { + fn new(client: &TonClient, cache_params: Option) -> Self { + let cache_params = cache_params.unwrap_or_default(); + + let cache = Cache::builder() + .max_capacity(cache_params.capacity) + .time_to_live(cache_params.time_to_live) + .build(); + + Self { + client: client.clone(), + cache, + } + } + + async fn get_libs( + &self, + lib_hashes: &[TonHash], + ) -> Result, TonLibraryError> { + let mut result_libs = HashMap::new(); + + let cached_libs_future = lib_hashes + .iter() + .map(|key| async move { (key, self.cache.get(key).await) }); + let maybe_cached_libs = join_all(cached_libs_future).await; + + let mut hashes_to_load = vec![]; + for (hash, value) in maybe_cached_libs { + match value { + Some(lib) => { + result_libs.insert(*hash, lib); + } + None => { + hashes_to_load.push(*hash); + } + } + } + + let blockchain_libs = self.load_libraries_impl(hashes_to_load.as_slice()).await?; + result_libs.extend(blockchain_libs.iter().map(|l| (l.cell_hash(), l.clone()))); + + self.insert_to_lib_cache(blockchain_libs).await?; + Ok(result_libs) + } + + async fn insert_to_lib_cache(&self, libs: Vec) -> Result<(), TonLibraryError> { + let mut cache_insert_futures = Vec::with_capacity(libs.len()); + for lib in libs { + cache_insert_futures.push(self.cache.insert(lib.cell_hash(), lib.clone())); + } + join_all(cache_insert_futures).await; + Ok(()) + } + + async fn load_libraries_impl( + &self, + hashes: &[TonHash], + ) -> Result, TonLibraryError> { + let library_list: Vec<_> = hashes + .iter() + .map(|hash| TonLibraryId::from(*hash)) + .collect(); + let library_result = self.client.smc_get_libraries(&library_list).await?; + + let libraries: Vec = library_result + .result + .into_iter() + .map(|lib| BagOfCells::parse(&lib.data)?.into_single_root()) + .collect::>()?; + + Ok(libraries) + } +} diff --git a/client/src/contract/factory/builder.rs b/client/src/contract/factory/builder.rs index c38ac377..4324503a 100644 --- a/client/src/contract/factory/builder.rs +++ b/client/src/contract/factory/builder.rs @@ -2,7 +2,7 @@ use std::sync::atomic::AtomicI32; use std::sync::Arc; use std::time::Duration; -use super::{BlockchainLibraryLoader, LibraryLoader, LibraryProvider}; +use super::{BlockchainLibraryProvider, LibraryProvider}; use crate::client::TonClient; use crate::contract::{TonContractError, TonContractFactory}; @@ -14,7 +14,7 @@ pub struct TonContractFactoryBuilder { txid_cache_capacity: u64, txid_cache_time_to_live: Duration, presync_blocks: i32, - library_provider: LibraryProvider, + library_provider: Arc, current_seqno: Arc, } @@ -29,8 +29,8 @@ impl TonContractFactoryBuilder { pub(crate) fn new(client: &TonClient) -> Self { let current_seqno_counter: Arc = Arc::new(0.into()); - let loader = BlockchainLibraryLoader::new(client); - let library_provider = LibraryProvider::new(loader, None, current_seqno_counter.clone()); + + let library_provider = Arc::new(BlockchainLibraryProvider::new(client, None)); TonContractFactoryBuilder { client: client.clone(), with_cache: true, @@ -87,14 +87,11 @@ impl TonContractFactoryBuilder { ) .await } - pub fn with_library_loader(&mut self, library_loader: &Arc) -> &mut Self { - let library_provider = - LibraryProvider::new(library_loader.clone(), None, self.current_seqno.clone()); - self.library_provider = library_provider; - self - } - pub fn with_library_provider(&mut self, library_provider: &LibraryProvider) -> &mut Self { + pub fn with_library_provider( + &mut self, + library_provider: Arc, + ) -> &mut Self { self.library_provider = library_provider.clone(); self } diff --git a/client/src/contract/factory/library_helper.rs b/client/src/contract/factory/library_helper.rs new file mode 100644 index 00000000..3b7a352a --- /dev/null +++ b/client/src/contract/factory/library_helper.rs @@ -0,0 +1,81 @@ +use std::collections::{HashMap, HashSet, VecDeque}; + +use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; +use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; +use tonlib_core::TonHash; + +use super::ContractLibraryDict; +use crate::contract::TonLibraryError; + +pub struct LibraryHelper; + +impl LibraryHelper { + pub fn store_to_dict( + library_hashes: &[TonHash], + lib_hashmap: HashMap, + ) -> Result { + let lib_cell = CellBuilder::new() + .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? + .build()?; + + let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; + let keys = library_hashes.to_vec(); + let dict = ContractLibraryDict { dict_boc, keys }; + Ok(dict) + } + + pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> + where + I: IntoIterator, + { + let mut arc_cell_buffer = VecDeque::new(); + + arc_cell_buffer.extend(cells); + + let mut lib_cells = HashSet::new(); + let mut visited_cells = HashSet::new(); + + while let Some(cell) = arc_cell_buffer.pop_front() { + if !visited_cells.insert(cell) { + continue; + } + let refs = cell.references(); + arc_cell_buffer.extend(refs.iter()); + if cell.is_library() { + lib_cells.insert(cell.clone()); + } + } + + let lib_hashes: Vec = lib_cells + .iter() + .map(|i| i.data()[1..].try_into()) + .collect::>()?; + + Ok(lib_hashes) + } +} + +#[cfg(test)] +mod tests { + + use tokio_test::assert_ok; + use tonlib_core::cell::BagOfCells; + use tonlib_core::TonHash; + + use crate::contract::LibraryHelper; + + #[test] + fn test_get_lib_hashes_by_code() -> anyhow::Result<()> { + let boc =hex::decode("b5ee9c72410101010023000842029f31f4f413a3accb706c88962ac69d59103b013a0addcfaeed5dd73c18fa98a866a5f879").unwrap(); + let expected_lib_id = + TonHash::from_hex("9f31f4f413a3accb706c88962ac69d59103b013a0addcfaeed5dd73c18fa98a8") + .unwrap(); + let code = BagOfCells::parse(&boc)?.into_single_root()?; + let hashes = assert_ok!(LibraryHelper::extract_library_hashes(&[code])); + + assert_eq!(hashes.len(), 1); + assert_eq!(expected_lib_id, hashes[0]); + + Ok(()) + } +} diff --git a/client/src/contract/factory/library_loader.rs b/client/src/contract/factory/library_loader.rs deleted file mode 100644 index 3e4f60d5..00000000 --- a/client/src/contract/factory/library_loader.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::sync::Arc; - -use async_trait::async_trait; -use tonlib_core::cell::{ArcCell, BagOfCells}; -use tonlib_core::TonHash; - -use crate::client::{TonClient, TonClientInterface}; -use crate::contract::TonLibraryError; -use crate::tl::TonLibraryId; - -#[derive(Debug, PartialEq)] -pub struct ContractLibraryDict { - pub dict_boc: Vec, - pub keys: Vec, -} - -#[async_trait] -pub trait LibraryLoader: Send + Sync { - async fn load_libraries( - &self, - hashes: &[TonHash], - seqno: Option, - ) -> Result, TonLibraryError>; -} - -pub struct BlockchainLibraryLoader { - client: TonClient, -} - -impl BlockchainLibraryLoader { - pub fn new(client: &TonClient) -> Arc { - Arc::new(BlockchainLibraryLoader { - client: client.clone(), - }) - } -} - -#[async_trait] -impl LibraryLoader for BlockchainLibraryLoader { - async fn load_libraries( - &self, - hashes: &[TonHash], - _seqno: Option, - ) -> Result, TonLibraryError> { - let mut results = Vec::new(); - - // If hashes exceed MAX_LIBS_REQUESTED, split them into chunks - for chunk in hashes.chunks(Self::MAX_LIBS_REQUESTED) { - let mut partial_result = self.load_libraries_impl(chunk).await?; - results.append(&mut partial_result); - } - Ok(results) - } -} - -impl BlockchainLibraryLoader { - const MAX_LIBS_REQUESTED: usize = 255; - - async fn load_libraries_impl( - &self, - hashes: &[TonHash], - ) -> Result, TonLibraryError> { - let library_list: Vec<_> = hashes - .iter() - .map(|hash| TonLibraryId::from(*hash)) - .collect(); - let library_result = self.client.smc_get_libraries(&library_list).await?; - - let libraries: Vec = library_result - .result - .into_iter() - .map(|lib| BagOfCells::parse(&lib.data)?.into_single_root()) - .collect::>()?; - - Ok(libraries) - } -} diff --git a/client/src/contract/factory/library_provider.rs b/client/src/contract/factory/library_provider.rs index 999f611d..2e5e46ed 100644 --- a/client/src/contract/factory/library_provider.rs +++ b/client/src/contract/factory/library_provider.rs @@ -1,229 +1,20 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::atomic::{AtomicI32, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use futures::future::join_all; -use moka::future::Cache; -use tonlib_core::cell::dict::predefined_writers::val_writer_ref_cell; -use tonlib_core::cell::{ArcCell, BagOfCells, CellBuilder}; +use async_trait::async_trait; +use tonlib_core::cell::ArcCell; use tonlib_core::TonHash; -use super::{ContractLibraryDict, LibraryLoader}; use crate::contract::TonLibraryError; -const DEFAULT_LIBRARY_CACHE_CAPACITY: u64 = 300; -const DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); -const DEFAULT_INTERNAL_CACHE_CAPACITY: u64 = 300; -const DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE: Duration = Duration::from_secs(60 * 60); - -const DEFAULT_LIBRARY_CACHE_PARAMS: LibraryCacheParams = LibraryCacheParams { - lib_cache_capacity: DEFAULT_LIBRARY_CACHE_CAPACITY, - lib_cache_time_to_live: DEFAULT_LIBRARY_CACHE_TIME_TO_LIVE, - internal_cache_capacity: DEFAULT_INTERNAL_CACHE_CAPACITY, - internal_cache_time_to_live: DEFAULT_INTERNAL_CACHE_TIME_TO_LIVE, -}; - -#[derive(Clone, Copy)] -pub struct LibraryCacheParams { - lib_cache_capacity: u64, - lib_cache_time_to_live: Duration, - internal_cache_capacity: u64, - internal_cache_time_to_live: Duration, -} - -#[derive(Clone)] -pub struct LibraryProvider { - inner: Arc, +#[derive(Debug, PartialEq)] +pub struct ContractLibraryDict { + pub dict_boc: Vec, + pub keys: Vec, } -struct Inner { - loader: Arc, - cache_by_hash_seqno: Cache<(TonHash, i32), Option>, - cache_by_hash: Cache, - current_seqno: Arc, -} - -impl LibraryProvider { - pub fn new( - loader: Arc, - cache_params: Option, - current_seqno: Arc, - ) -> Self { - Self { - inner: Arc::new(Inner::new(loader, cache_params, current_seqno)), - } - } - - pub fn extract_library_hashes<'a, I>(cells: I) -> Result, TonLibraryError> - where - I: IntoIterator, - { - let mut arc_cell_buffer = VecDeque::new(); - - arc_cell_buffer.extend(cells); - - let mut lib_cells = HashSet::new(); - let mut visited_cells = HashSet::new(); - - while let Some(cell) = arc_cell_buffer.pop_front() { - if !visited_cells.insert(cell) { - continue; - } - let refs = cell.references(); - arc_cell_buffer.extend(refs.iter()); - if cell.is_library() { - lib_cells.insert(cell.clone()); - } - } - - let lib_hashes: Vec = lib_cells - .iter() - .map(|i| i.data()[1..].try_into()) - .collect::>()?; - - Ok(lib_hashes) - } - - pub async fn get_libs_by_seqno<'a, I>( - &self, - cells: I, - seqno: i32, - ) -> Result - where - I: IntoIterator, - { - let lib_hashes = LibraryProvider::extract_library_hashes(cells)?; - let libs = self.inner.get_libs(lib_hashes.as_slice(), seqno).await?; - LibraryProvider::store_to_dict(lib_hashes.as_slice(), libs) - } - - pub async fn get_libs_latest<'a, I>( +#[async_trait] +pub trait LibraryProvider: Send + Sync { + async fn get_libs( &self, - cells: I, - ) -> Result - where - I: IntoIterator, - { - let seqno = self.inner.current_seqno.load(Ordering::Relaxed); - self.get_libs_by_seqno(cells, seqno).await - } - - fn store_to_dict( - library_hashes: &[TonHash], - lib_hashmap: HashMap, - ) -> Result { - let lib_cell = CellBuilder::new() - .store_dict_data(256, val_writer_ref_cell, lib_hashmap)? - .build()?; - - let dict_boc = BagOfCells::from_root(lib_cell).serialize(false)?; - let keys = library_hashes.to_vec(); - let dict = ContractLibraryDict { dict_boc, keys }; - Ok(dict) - } -} - -impl Inner { - pub(crate) fn new( - loader: Arc, - cache_params: Option, - current_seqno: Arc, - ) -> Self { - let cache_params = cache_params.unwrap_or(DEFAULT_LIBRARY_CACHE_PARAMS); - - let library_cache = Cache::builder() - .max_capacity(cache_params.lib_cache_capacity) - .time_to_live(cache_params.lib_cache_time_to_live) - .build(); - let internal_cache = Cache::builder() - .max_capacity(cache_params.internal_cache_capacity) - .time_to_live(cache_params.internal_cache_time_to_live) - .build(); - Self { - loader, - cache_by_hash_seqno: library_cache, - cache_by_hash: internal_cache, - current_seqno, - } - } - - pub(crate) async fn get_libs( - &self, - lib_hashes: &[TonHash], - seqno: i32, - ) -> Result, TonLibraryError> -where { - let mut result_libs = HashMap::new(); - - let keys = lib_hashes - .iter() - .map(|h| (*h, seqno)) - .collect::>(); - let cached_libs_future = keys - .iter() - .map(|key| async move { (key.0, self.cache_by_hash_seqno.get(key).await) }); - let maybe_cached_libs = join_all(cached_libs_future).await; - - let mut hashes_to_load = vec![]; - for (hash, value) in maybe_cached_libs { - // outer option means whether library is in the cache or not - // inner option means whether library is available at certain seqno - match value { - Some(Some(lib)) => { - result_libs.insert(hash, lib); - } - Some(None) => {} - None => { - log::trace!("loading lib from BC: {:?}", hash); - hashes_to_load.push(hash); - } - } - } - - // load libs from blockchain - let mut blockchain_libs = self - .loader - .load_libraries(hashes_to_load.as_slice(), Some(seqno)) - .await?; - self.replace_by_existing_data(&mut blockchain_libs).await; - result_libs.extend(blockchain_libs.iter().map(|l| (l.cell_hash(), l.clone()))); - - self.insert_to_lib_cache(blockchain_libs, seqno).await?; - Ok(result_libs) - } - - async fn insert_to_lib_cache( - &self, - libs: Vec, - seqno: i32, - ) -> Result<(), TonLibraryError> { - let mut cache_insert_futures = Vec::with_capacity(libs.len()); - let mut internal_cache_insert_futures = Vec::with_capacity(libs.len()); - for lib in libs { - cache_insert_futures.push( - self.cache_by_hash_seqno - .insert((lib.cell_hash(), seqno), Some(lib.clone())), - ); - internal_cache_insert_futures.push(self.cache_by_hash.insert(lib.cell_hash(), lib)); - } - - join_all(cache_insert_futures).await; - join_all(internal_cache_insert_futures).await; - Ok(()) - } - - async fn replace_by_existing_data(&self, loaded_libs: &mut [ArcCell]) { - let loaded_libs_keys: Vec<_> = loaded_libs.iter().map(|l| l.cell_hash()).collect(); - let future_internals = loaded_libs_keys.iter().map(|k| self.cache_by_hash.get(k)); - - let internal_cached_libs = join_all(future_internals).await; - - // replace loaded libs from blockchain to ones from internal cache. - for i in 0..loaded_libs.len() { - if let Some(lib) = &internal_cached_libs[i] { - loaded_libs[i] = lib.clone() - } - } - } + cells: &[ArcCell], + mc_seqno: Option, + ) -> Result; } diff --git a/client/src/contract/state.rs b/client/src/contract/state.rs index 9c6c7870..95f5dbe6 100644 --- a/client/src/contract/state.rs +++ b/client/src/contract/state.rs @@ -102,7 +102,7 @@ impl TonContractState { let libs = self .factory .library_provider() - .get_libs_latest(&[code, data]) + .get_libs(&[code, data], None) .await?; let run_result = unsafe { diff --git a/client/tests/contract_emulator_test.rs b/client/tests/contract_emulator_test.rs index 22a80d46..eabad039 100644 --- a/client/tests/contract_emulator_test.rs +++ b/client/tests/contract_emulator_test.rs @@ -244,7 +244,7 @@ async fn benchmark_emulate_ston_router_v2() -> anyhow::Result<()> { )?; let libs = factory .library_provider() - .get_libs_by_seqno(&[code_cell, data_cell], 47497716) + .get_libs(&[code_cell, data_cell], None) .await?; let mut sums: ((Duration, Duration, Duration, Duration), Duration) = ( @@ -358,7 +358,7 @@ async fn test_lib_cache_works() -> anyhow::Result<()> { let t = Instant::now(); let libs = factory .library_provider() - .get_libs_latest(&[code_cell.clone(), data_cell.clone()]) + .get_libs(&[code_cell.clone(), data_cell.clone()], None) .await?; if t.elapsed() > Duration::from_millis(10) { diff --git a/client/tests/library_test.rs b/client/tests/library_test.rs index 9aeb24f9..cd442a84 100644 --- a/client/tests/library_test.rs +++ b/client/tests/library_test.rs @@ -1,11 +1,6 @@ -use std::str::FromStr; - -use tokio_test::assert_ok; -use tonlib_client::contract::{ - BlockchainLibraryLoader, LibraryLoader, LibraryProvider, TonContractFactory, -}; +use tonlib_client::contract::BlockchainLibraryProvider; use tonlib_core::cell::BagOfCells; -use tonlib_core::{TonAddress, TonHash}; +use tonlib_core::TonHash; mod common; @@ -20,38 +15,17 @@ async fn test_get_libs_by_hash() -> anyhow::Result<()> { ]); log::info!("{:?}", expected_lib_id); - let library_loader = BlockchainLibraryLoader::new(&client); + let library_loader = BlockchainLibraryProvider::new(&client, None); let maybe_lib = library_loader - .load_libraries(&[expected_lib_id, expected_lib_id], None) + .load_libraries(&[expected_lib_id, expected_lib_id]) .await?; + let boc = BagOfCells::from_root(maybe_lib[0].as_ref().clone()) + .serialize(true) + .unwrap(); + log::info!("{}", hex::encode(boc)); + assert_eq!(maybe_lib.len(), 1); assert_eq!(maybe_lib[0].cell_hash(), expected_lib_id); Ok(()) } - -#[tokio::test] -async fn test_get_lib_hashes_by_code() -> anyhow::Result<()> { - common::init_logging(); - let client = common::new_mainnet_client().await; - let factory = TonContractFactory::builder(&client).build().await?; - - let address = TonAddress::from_str("EQCqX53C_Th32Xg7UyrlqF0ypmePjljxG8edlwfT-1QpG3TB")?; - - let state = factory.get_latest_account_state(&address).await?; - let code = BagOfCells::parse(&state.code)?.into_single_root()?; - - let hashes = assert_ok!(LibraryProvider::extract_library_hashes(&[code])); - - log::info!("{:?}", hashes); - - let expected_lib_id = TonHash::from([ - 159, 49, 244, 244, 19, 163, 172, 203, 112, 108, 136, 150, 42, 198, 157, 89, 16, 59, 1, 58, - 10, 221, 207, 174, 237, 93, 215, 60, 24, 250, 152, 168, - ]); - - assert_eq!(hashes.len(), 1); - assert_eq!(expected_lib_id, hashes[0]); - - Ok(()) -} From 2a37aabcb8639d30d573a90f432feba365ddcf71 Mon Sep 17 00:00:00 2001 From: Dmitrii Baranov Date: Thu, 9 Jan 2025 12:19:49 +0000 Subject: [PATCH 17/18] NI: bump tonlib-sys to 2024.10.4 --- Cargo.toml | 6 +++--- client/README.md | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 81a18f66..44cb8925 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" [workspace.package] -version = "0.22.4" +version = "0.23.0" edition = "2021" description = "Rust SDK for The Open Network" license = "MIT" @@ -49,10 +49,10 @@ tokio-retry = "0.3" tokio-test = "0.4" ton_liteapi = "0.1.0" adnl = "2.0" -tonlib-sys = "=2024.10.3" +tonlib-sys = "=2024.10.4" tokio-tower = "0.6.0" tower = "0.5.1" libc = "0.2" # internal deps -tonlib-core = { version = "0.22", path = "core" } +tonlib-core = { version = "0.23", path = "core" } diff --git a/client/README.md b/client/README.md index 3b4815c1..7e36d55b 100644 --- a/client/README.md +++ b/client/README.md @@ -140,8 +140,7 @@ async fn call_blockchain_methods()-> anyhow::Result<()>{ txs.incomplete ); for tx_id in txs.transactions { - let mut t: TonHash = ZERO_HASH; - t.clone_from_slice(tx_id.account.as_slice()); + let t = TonHash::try_from(tx_id.account.as_slice())?; let addr = TonAddress::new(workchain, &t); let id = InternalTransactionId { hash: tx_id.hash.clone(), From dd1d66c2e8b94b6bc0b7c7f5f8f08eeb2bc0cba4 Mon Sep 17 00:00:00 2001 From: dbaranov34 Date: Thu, 9 Jan 2025 16:26:12 +0400 Subject: [PATCH 18/18] Release v0.23.0 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a85988b8..e0ff98fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,3 +64,11 @@ ### v0.22.4 * make pub TvmEmulatorUnsafe +### v0.23.0 + +* Impl #BE-1846: library cache +* Impl #BE-1881: Replaced dashmap by tokio::sync:Mutex to avoid mem leaks +* Impl #BE-1839: Boxed long errors, updated builder methods +* Impl #BE-1892: Display for TonHash +* Impl #BE-1893: Library provider trait and implementation of BlockChainLibraryProvider +* NI: bump tonlib-sys to 2024.10.4