diff --git a/pkg/addressbook/addressbook.go b/pkg/addressbook/addressbook.go index 4efd3f98..176b366b 100644 --- a/pkg/addressbook/addressbook.go +++ b/pkg/addressbook/addressbook.go @@ -3,11 +3,6 @@ package addressbook import ( "encoding/json" "fmt" - "github.com/tonkeeper/opentonapi/pkg/core" - imgGenerator "github.com/tonkeeper/opentonapi/pkg/image" - "github.com/tonkeeper/tongo/abi" - "github.com/tonkeeper/tongo/tlb" - "github.com/tonkeeper/tongo/ton" "io" "math/rand" "net/http" @@ -17,6 +12,12 @@ import ( "sync" "time" + "github.com/tonkeeper/opentonapi/pkg/core" + imgGenerator "github.com/tonkeeper/opentonapi/pkg/image" + "github.com/tonkeeper/tongo/abi" + "github.com/tonkeeper/tongo/tlb" + "github.com/tonkeeper/tongo/ton" + "github.com/shopspring/decimal" "github.com/tonkeeper/tongo" "go.uber.org/zap" @@ -27,7 +28,9 @@ import ( "github.com/tonkeeper/opentonapi/pkg/oas" ) -// KnownAddress represents additional manually crafted information about a particular account in the blockchain. +var NormalizeReg = regexp.MustCompile("[^\\p{L}\\p{N}]") + +// KnownAddress represents additional manually crafted information about a particular account in the blockchain type KnownAddress struct { IsScam bool `json:"is_scam,omitempty"` RequireMemo bool `json:"require_memo,omitempty"` @@ -36,6 +39,7 @@ type KnownAddress struct { Image string `json:"image,omitempty"` } +// AttachedAccountType defines different types of accounts (e.g., manual, NFT) type AttachedAccountType string const ( @@ -53,6 +57,7 @@ type AttachedAccount struct { Name string `json:"name"` Preview string `json:"preview"` Wallet ton.AccountID `json:"wallet"` + Slug string `json:"-"` Symbol string `json:"-"` Type AttachedAccountType `json:"-"` Weight int32 `json:"-"` @@ -60,7 +65,7 @@ type AttachedAccount struct { Normalized string `json:"-"` } -// KnownJetton represents additional manually crafted information about a particular jetton in the blockchain. +// KnownJetton represents additional manually crafted information about a particular jetton in the blockchain type KnownJetton struct { Name string `json:"name"` Verification core.TrustType `json:"verification"` @@ -75,7 +80,7 @@ type KnownJetton struct { Coingecko string `json:"coingecko,omitempty"` } -// KnownCollection represents additional manually crafted information about a particular NFT collection in the blockchain. +// KnownCollection represents additional manually crafted information about a particular NFT collection in the blockchain type KnownCollection struct { Address string `json:"address"` Approvers []oas.NftApprovedByItem @@ -102,7 +107,7 @@ func WithAdditionalAddressesSource(a addresser) Option { } } -// Book holds information about known accounts, jettons, NFT collections manually crafted by the tonkeeper team and the community. +// Book holds information about known accounts, jettons, NFT collections manually crafted by the tonkeeper team and the community type Book struct { addressers []addresser @@ -114,12 +119,14 @@ type Book struct { walletsResolved cache.Cache[tongo.AccountID, bool] } +// TFPoolInfo holds information about a token pool type TFPoolInfo struct { Name string `json:"name"` GroupName string `json:"groupName"` Address string `json:"address"` } +// GetAddressInfoByAddress fetches address info if available func (b *Book) GetAddressInfoByAddress(a tongo.AccountID) (KnownAddress, bool) { for i := range b.addressers { if a1, ok := b.addressers[i].GetAddress(a); ok { @@ -129,8 +136,9 @@ func (b *Book) GetAddressInfoByAddress(a tongo.AccountID) (KnownAddress, bool) { return KnownAddress{}, false } +// SearchAttachedAccountsByPrefix searches for accounts by prefix func (b *Book) SearchAttachedAccountsByPrefix(prefix string) []AttachedAccount { - prefix = strings.ToLower(normalizeReg.ReplaceAllString(prefix, "")) + prefix = strings.ToLower(NormalizeReg.ReplaceAllString(prefix, "")) var accounts []AttachedAccount for i := range b.addressers { foundAccounts := b.addressers[i].SearchAttachedAccounts(prefix) @@ -140,12 +148,13 @@ func (b *Book) SearchAttachedAccountsByPrefix(prefix string) []AttachedAccount { } tonDomainPrefix := prefix + "ton" tgDomainPrefix := prefix + "tme" - + // Adjust weight for full matches for i := range accounts { if accounts[i].Normalized == prefix || accounts[i].Normalized == tonDomainPrefix || accounts[i].Normalized == tgDomainPrefix { - accounts[i].Weight *= 100 // full match + accounts[i].Weight *= 100 } } + // Sort and limit the result sort.Slice(accounts, func(i, j int) bool { if accounts[i].Weight == accounts[j].Weight { return len(accounts[i].Name) < len(accounts[j].Name) @@ -158,6 +167,7 @@ func (b *Book) SearchAttachedAccountsByPrefix(prefix string) []AttachedAccount { return accounts } +// GetTFPoolInfo retrieves token pool info for an account func (b *Book) GetTFPoolInfo(a tongo.AccountID) (TFPoolInfo, bool) { b.mu.RLock() defer b.mu.RUnlock() @@ -165,6 +175,7 @@ func (b *Book) GetTFPoolInfo(a tongo.AccountID) (TFPoolInfo, bool) { return info, ok } +// GetKnownCollections returns all known collections func (b *Book) GetKnownCollections() map[tongo.AccountID]KnownCollection { b.mu.RLock() defer b.mu.RUnlock() @@ -176,6 +187,7 @@ func (b *Book) GetKnownCollections() map[tongo.AccountID]KnownCollection { return collections } +// GetCollectionInfoByAddress retrieves collection info for a specific address func (b *Book) GetCollectionInfoByAddress(a tongo.AccountID) (KnownCollection, bool) { b.mu.RLock() defer b.mu.RUnlock() @@ -183,12 +195,14 @@ func (b *Book) GetCollectionInfoByAddress(a tongo.AccountID) (KnownCollection, b return c, ok } +// GetKnownJettons returns all known jettons func (b *Book) GetKnownJettons() map[tongo.AccountID]KnownJetton { b.mu.RLock() defer b.mu.RUnlock() return maps.Clone(b.jettons) } +// GetJettonInfoByAddress fetches jetton info for a specific address func (b *Book) GetJettonInfoByAddress(a tongo.AccountID) (KnownJetton, bool) { b.mu.RLock() defer b.mu.RUnlock() @@ -202,13 +216,14 @@ func (b *Book) GetJettonInfoByAddress(a tongo.AccountID) (KnownJetton, bool) { return j, ok } +// TFPools returns a list of all token pools func (b *Book) TFPools() []tongo.AccountID { b.mu.RLock() defer b.mu.RUnlock() return maps.Keys(b.tfPools) } -// IsWallet returns true if the given address is a wallet. +// IsWallet checks if the address is a wallet func (b *Book) IsWallet(addr tongo.AccountID) (bool, error) { if wallet, ok := b.walletsResolved.Get(addr); ok { return wallet, nil @@ -218,7 +233,7 @@ func (b *Book) IsWallet(addr tongo.AccountID) (bool, error) { return false, fmt.Errorf("failed to figure out if %v is a wallet %w", addr, err) } if status == tlb.AccountNone || status == tlb.AccountUninit { - b.walletsResolved.Set(addr, true, cache.WithExpiration(time.Minute)) //shorter period for non-existing accounts + b.walletsResolved.Set(addr, true, cache.WithExpiration(time.Minute)) // Cache short for non-existing accounts return true, nil } isWallet := false @@ -232,45 +247,49 @@ func (b *Book) IsWallet(addr tongo.AccountID) (bool, error) { return isWallet, nil } -type manyalAddresser struct { +type manualAddresser struct { mu sync.RWMutex addresses map[tongo.AccountID]KnownAddress sorted []AttachedAccount } -func (m *manyalAddresser) GetAddress(a tongo.AccountID) (KnownAddress, bool) { +// GetAddress fetches known address by account +func (m *manualAddresser) GetAddress(a tongo.AccountID) (KnownAddress, bool) { m.mu.RLock() defer m.mu.RUnlock() addr, ok := m.addresses[a] return addr, ok } -func (m *manyalAddresser) SearchAttachedAccounts(prefix string) []AttachedAccount { +// SearchAttachedAccounts searches for accounts by prefix +func (m *manualAddresser) SearchAttachedAccounts(prefix string) []AttachedAccount { m.mu.RLock() sortedList := m.sorted m.mu.RUnlock() - startIdx, endIdx := findIndexes(sortedList, prefix) + + if len(sortedList) == 0 { + return []AttachedAccount{} + } + // Normalize the prefix for comparison + prefix = strings.ToLower(NormalizeReg.ReplaceAllString(prefix, "")) + startIdx, endIdx := FindIndexes(sortedList, prefix) if startIdx == -1 || endIdx == -1 { - return nil + return []AttachedAccount{} } + // Return the found accounts as a slice foundAccounts := make([]AttachedAccount, endIdx-startIdx+1) copy(foundAccounts, sortedList[startIdx:endIdx+1]) return foundAccounts } -var normalizeReg = regexp.MustCompile("[^\\p{L}\\p{N}]") - -func (m *manyalAddresser) refreshAddresses(addressPath string) error { +// refreshAddresses updates the list of known addresses +func (m *manualAddresser) refreshAddresses(addressPath string) error { addresses, err := downloadJson[KnownAddress](addressPath) if err != nil { return err } newAddresses := make(map[tongo.AccountID]KnownAddress, len(addresses)) - newSorted := []AttachedAccount{ - {Name: "The Locker", Wallet: ton.MustParseAccountID("EQDtFpEwcFAEcRe5mLVh2N6C0x-_hJEM7W61_JLnSF74p4q2"), Normalized: "locker", Weight: 1000}, - {Name: "The Locker", Wallet: ton.MustParseAccountID("EQDtFpEwcFAEcRe5mLVh2N6C0x-_hJEM7W61_JLnSF74p4q2"), Normalized: "thelocker", Weight: 10000}, - {Name: "Ton Foundation", Wallet: ton.MustParseAccountID("EQCLyZHP4Xe8fpchQz76O-_RmUhaVc_9BAoGyJrwJrcbz2eZ"), Normalized: "foundation", Weight: 1000}, - } + var newSorted []AttachedAccount for _, item := range addresses { account, err := tongo.ParseAddress(item.Address) if err != nil { @@ -282,31 +301,52 @@ func (m *manyalAddresser) refreshAddresses(addressPath string) error { if item.Image != "" { preview = imgGenerator.DefaultGenerator.GenerateImageUrl(item.Image, 200, 200) } - newSorted = append(newSorted, AttachedAccount{ - Name: item.Name, - Preview: preview, - Wallet: account.ID, - Type: ManualAccountType, - Weight: 1000, - Popular: 1, - Normalized: strings.ToLower(normalizeReg.ReplaceAllString(item.Name, "")), - }) + names := GenerateNameVariants(item.Name) + for idx, name := range names { + // Assign initial weight, give extra weight to the first name for priority + weight := int32(1000) + if idx == 0 { + weight *= 10 // Boost weight for the first name + } + newSorted = append(newSorted, AttachedAccount{ + Name: item.Name, + Preview: preview, + Wallet: account.ID, + Type: ManualAccountType, + Weight: weight, + Popular: 1, + Normalized: strings.ToLower(NormalizeReg.ReplaceAllString(name, "")), + }) + } } sort.Slice(newSorted, func(i, j int) bool { return newSorted[i].Normalized < newSorted[j].Normalized }) + m.mu.Lock() - defer m.mu.Unlock() m.addresses = newAddresses m.sorted = newSorted + m.mu.Unlock() + return nil } -func findIndexes(sortedList []AttachedAccount, prefix string) (int, int) { - low := 0 - high := len(sortedList) - 1 +func GenerateNameVariants(name string) []string { + words := strings.Fields(name) // Split the name into words + var variants []string + // Generate up to 3 variants by rotating the words + for i := 0; i < len(words) && i < 3; i++ { + variant := append(words[i:], words[:i]...) // Rotate the words + variants = append(variants, strings.Join(variant, " ")) + } + return variants +} + +func FindIndexes(sortedList []AttachedAccount, prefix string) (int, int) { + low, high := 0, len(sortedList)-1 startIdx := -1 - for low <= high { // Find the starting index + // Find starting index for the prefix + for low <= high { med := (low + high) / 2 if strings.HasPrefix(sortedList[med].Normalized, prefix) { startIdx = med @@ -317,15 +357,13 @@ func findIndexes(sortedList []AttachedAccount, prefix string) (int, int) { high = med - 1 } } - - if startIdx == -1 { // Prefix not found + if startIdx == -1 { // No prefix match return -1, -1 } - - low = startIdx - high = len(sortedList) - 1 + low, high = startIdx, len(sortedList)-1 endIdx := -1 - for low <= high { // Find the ending index + // Find ending index for the prefix + for low <= high { med := (low + high) / 2 if strings.HasPrefix(sortedList[med].Normalized, prefix) { endIdx = med @@ -338,14 +376,16 @@ func findIndexes(sortedList []AttachedAccount, prefix string) (int, int) { return startIdx, endIdx } +// NewAddressBook initializes a Book and starts background refreshers tasks func NewAddressBook(logger *zap.Logger, addressPath, jettonPath, collectionPath string, storage accountsStatesSource, opts ...Option) *Book { - var manual = &manyalAddresser{ + var manual = &manualAddresser{ addresses: make(map[tongo.AccountID]KnownAddress), } options := Options{addressers: []addresser{manual}} for _, opt := range opts { opt(&options) } + collections := make(map[tongo.AccountID]KnownCollection) jettons := make(map[tongo.AccountID]KnownJetton) tfPools := make(map[tongo.AccountID]TFPoolInfo) @@ -358,17 +398,18 @@ func NewAddressBook(logger *zap.Logger, addressPath, jettonPath, collectionPath addressers: options.addressers, walletsResolved: cache.NewLRUCache[tongo.AccountID, bool](1_000_000, "is_wallet"), } - - go refresher("gg whitelist", time.Hour, 5*time.Minute, logger, book.getGGWhitelist) - go refresher("addresses", time.Minute*15, 5*time.Minute, logger, func() error { return manual.refreshAddresses(addressPath) }) - go refresher("jettons", time.Minute*15, 5*time.Minute, logger, func() error { return book.refreshJettons(jettonPath) }) - go refresher("collections", time.Minute*15, 5*time.Minute, logger, func() error { return book.refreshCollections(collectionPath) }) - book.refreshTfPools(logger) //hardcoded so don't need to be refreshed + // Start background refreshers + go Refresher("gg whitelist", time.Hour, 5*time.Minute, logger, book.getGGWhitelist) + go Refresher("addresses", time.Minute*15, 5*time.Minute, logger, func() error { return manual.refreshAddresses(addressPath) }) + go Refresher("jettons", time.Minute*15, 5*time.Minute, logger, func() error { return book.refreshJettons(jettonPath) }) + go Refresher("collections", time.Minute*15, 5*time.Minute, logger, func() error { return book.refreshCollections(collectionPath) }) + book.refreshTfPools(logger) // Refresh tfPools once on initialization as it doesn't need periodic updates return book } -func refresher(name string, interval, errorInterval time.Duration, logger *zap.Logger, f func() error) { +// Refresher periodically calls the provided function at the specified interval +func Refresher(name string, interval, errorInterval time.Duration, logger *zap.Logger, f func() error) { for { err := f() if err != nil { @@ -376,10 +417,12 @@ func refresher(name string, interval, errorInterval time.Duration, logger *zap.L time.Sleep(errorInterval + time.Duration(rand.Intn(10))*time.Second) continue } + // Wait for the next interval before refreshing again time.Sleep(interval + time.Duration(rand.Intn(10))*time.Second) } } +// refreshJettons fetches and updates the jetton data from the provided URL func (b *Book) refreshJettons(jettonPath string) error { jettons, err := downloadJson[KnownJetton](jettonPath) if err != nil { @@ -387,6 +430,7 @@ func (b *Book) refreshJettons(jettonPath string) error { } b.mu.Lock() defer b.mu.Unlock() + // Update jettons map with the fetched data for _, item := range jettons { account, err := tongo.ParseAddress(item.Address) if err != nil { @@ -398,6 +442,7 @@ func (b *Book) refreshJettons(jettonPath string) error { return nil } +// refreshCollections fetches and updates collection data from the provided URL func (b *Book) refreshCollections(collectionPath string) error { collections, err := downloadJson[KnownCollection](collectionPath) if err != nil { @@ -405,21 +450,22 @@ func (b *Book) refreshCollections(collectionPath string) error { } b.mu.Lock() defer b.mu.Unlock() + // Update collections map with the fetched data for _, item := range collections { - // TODO: remove items that were previously added but aren't present in the current list. + // TODO: remove items that were previously added but aren't present in the current list account, err := tongo.ParseAddress(item.Address) if err != nil { continue } currentCollection, ok := b.collections[account.ID] if !ok { - // this is a new item, so we only add tonkeeper as approver. + // Add new collection with Tonkeeper as the approver currentCollection.Address = account.ID.ToRaw() currentCollection.Approvers = []oas.NftApprovedByItem{oas.NftApprovedByItemTonkeeper} b.collections[account.ID] = currentCollection continue } - // this is an existing item, so we merge approvers and remove duplicates adding tonkeeper. + // Merge approvers and ensure Tonkeeper is included if !slices.Contains(currentCollection.Approvers, oas.NftApprovedByItemTonkeeper) { currentCollection.Approvers = append(item.Approvers, oas.NftApprovedByItemTonkeeper) b.collections[account.ID] = currentCollection @@ -428,10 +474,11 @@ func (b *Book) refreshCollections(collectionPath string) error { return nil } +// refreshTfPools fetches and updates the TF pool data func (b *Book) refreshTfPools(logger *zap.Logger) { b.mu.Lock() defer b.mu.Unlock() - + // Fetch and update the TF pools for _, pool := range getPools(logger) { account, err := tongo.ParseAddress(pool.Address) if err != nil { @@ -443,6 +490,7 @@ func (b *Book) refreshTfPools(logger *zap.Logger) { } } +// downloadJson is a utility function that downloads and unmarshals JSON data from a URL into a slice of the specified type func downloadJson[T any](url string) ([]T, error) { response, err := http.Get(url) if err != nil { @@ -463,6 +511,7 @@ func downloadJson[T any](url string) ([]T, error) { return data, nil } +// getGGWhitelist fetches the whitelist from the GetGems API and updates the collections func (b *Book) getGGWhitelist() error { addresses, err := fetchGetGemsVerifiedCollections() if err != nil { @@ -470,14 +519,17 @@ func (b *Book) getGGWhitelist() error { } b.mu.Lock() defer b.mu.Unlock() + // Update collections with new addresses from the whitelist for _, account := range addresses { collection, ok := b.collections[account] if !ok { + // Add new collection with Getgems as the approver collection.Address = account.ToRaw() collection.Approvers = []oas.NftApprovedByItem{oas.NftApprovedByItemGetgems} b.collections[account] = collection continue } + // Ensure Getgems is included as an approver if !slices.Contains(collection.Approvers, oas.NftApprovedByItemGetgems) { collection.Approvers = append(collection.Approvers, oas.NftApprovedByItemGetgems) b.collections[account] = collection @@ -486,6 +538,7 @@ func (b *Book) getGGWhitelist() error { return nil } +// fetchGetGemsVerifiedCollections fetches verified collections from GetGems API func fetchGetGemsVerifiedCollections() ([]tongo.AccountID, error) { res, err := downloadJson[string]("https://api.getgems.io/public/api/verified-collections") if err != nil { diff --git a/pkg/addressbook/addressbook_test.go b/pkg/addressbook/addressbook_test.go index 0d4d6361..61634db5 100644 --- a/pkg/addressbook/addressbook_test.go +++ b/pkg/addressbook/addressbook_test.go @@ -1,12 +1,153 @@ package addressbook import ( + "strings" "testing" + "time" "github.com/stretchr/testify/require" + "github.com/tonkeeper/opentonapi/pkg/config" + "github.com/tonkeeper/opentonapi/pkg/litestorage" + "github.com/tonkeeper/tongo/liteapi" "github.com/tonkeeper/tongo/ton" + "go.uber.org/zap" ) +func TestNormalizeString(t *testing.T) { + type testCase struct { + name string + input string + output string + } + tests := []testCase{ + { + name: "simple English sentence", + input: "The day that will always", + output: "thedaythatwillalways", + }, + { + name: "mixed case English words", + input: "TON Panda Baby", + output: "tonpandababy", + }, + { + name: "English with emoji at the end", + input: "apple 👎", + output: "apple", + }, + { + name: "emoji at the start", + input: "👎banana", + output: "banana", + }, + { + name: "uppercase Cyrillic", + input: "ХАЗЯЕВА", + output: "хазяева", + }, + { + name: "string with numbers and dots", + input: "11313.ton", + output: "11313ton", + }, + { + name: "Cyrillic sentence with spaces", + input: "Пока что думаю ", + output: "покачтодумаю", + }, + { + name: "English with multiple emojis", + input: "💎 TON Earth 🌍 Collectibles 🧭", + output: "tonearthcollectibles", + }, + { + name: "Cyrillic with numbers", + input: "Привет122", + output: "привет122", + }, + { + name: "string with numbers and domain-like structure", + input: "11313.t.me", + output: "11313tme", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + convertedData := strings.ToLower(NormalizeReg.ReplaceAllString(test.input, "")) + require.Equal(t, test.output, convertedData) + }) + } +} + +func TestSearchAttachedAccountsByPrefix(t *testing.T) { + logger, _ := zap.NewDevelopment() + client, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet()) + require.NoError(t, err, "Failed to create lite API client") + + liteStorage, err := litestorage.NewLiteStorage(logger, client) + require.NoError(t, err, "Failed to create lite storage") + + book := NewAddressBook(logger, config.AddressPath, config.JettonPath, config.CollectionPath, liteStorage) + + // Waiting background processes + time.Sleep(time.Second) + + tests := []struct { + name string + request string + expected *AttachedAccount + }{ + { + name: "Exact match with full name", + request: "TON Believers Fund", + expected: &AttachedAccount{ + Name: "TON Believers Fund", + Wallet: ton.MustParseAccountID("0:ed1691307050047117b998b561d8de82d31fbf84910ced6eb5fc92e7485ef8a7"), + }, + }, + { + name: "Partial match", + request: "believers fund", + expected: &AttachedAccount{ + Name: "TON Believers Fund", + Wallet: ton.MustParseAccountID("0:ed1691307050047117b998b561d8de82d31fbf84910ced6eb5fc92e7485ef8a7"), + }, + }, + { + name: "Single word partial match", + request: "fund", + expected: &AttachedAccount{ + Name: "TON Believers Fund", + Wallet: ton.MustParseAccountID("0:ed1691307050047117b998b561d8de82d31fbf84910ced6eb5fc92e7485ef8a7"), + }, + }, + { + name: "No match", + request: "random string", + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + accounts := book.SearchAttachedAccountsByPrefix(test.request) + if test.expected == nil { + require.Empty(t, accounts, "Expected no results, but got: %v", len(accounts)) + return + } + // Check if the expected account is in the result + var found bool + for _, account := range accounts { + if account.Wallet == test.expected.Wallet { + found = true + break + } + } + require.True(t, found, "Expected account not found in the results: %v", test.expected) + }) + } +} + func TestFetchGetGemsVerifiedCollections(t *testing.T) { accountIDs, err := fetchGetGemsVerifiedCollections() require.Nil(t, err)