Merge pull request #8997 from DefiLlama/aethir

CG bridged TVL coins

coins/coins2.ts

@@ -2,7 +2,7 @@ import { CoinRead } from "./src/adapters/utils/dbInterfaces";
 import getTVLOfRecordClosestToTimestamp from "./src/utils/shared/getRecordClosestToTimestamp";
 import { getCurrentUnixTimestamp } from "./src/utils/date";
 import { Redis } from "ioredis";
-import { getCoins2Connection } from "./getDBConnection";
+import { getCoins2Connection } from "./src/utils/shared/getDBConnection";
 import { sendMessage } from "../defi/src/utils/discord";
 import fetch from "node-fetch";
 
@@ -23,7 +23,7 @@ const latency: number = 1 * 60 * 60; // 1hr
 const margin: number = 6 * 60 * 60; // 12hr
 const confidenceThreshold: number = 0.3;
 const zeroDecimalAdapters: string[] = [
-  'LiNEAR',
+  "LiNEAR",
   "coingecko",
   "chainlink-nft",
   "defichain",
@@ -175,7 +175,7 @@ export async function getRedisConnection() {
     } catch (e) {
       reject(e);
     }
-  })
+  });
   return _redis;
 }
 
@@ -184,7 +184,7 @@ process.on("exit", async () => {
     const redis = await _redis;
     redis.quit();
   }
-})
+});
 
 export async function queryRedis(
   values: CoinRead[],
@@ -194,7 +194,7 @@ export async function queryRedis(
   const keys: string[] = values.map((v: CoinRead) => v.key);
   // console.log(`${values.length} queried`);
 
-  const redis = await getRedisConnection()
+  const redis = await getRedisConnection();
   let res = await redis.mget(keys);
   // console.log("mget finished");
   const jsonValues: { [key: string]: Coin } = {};
@@ -529,7 +529,7 @@ export async function writeToRedis(
     );
   }
   try {
-    const redis = await getRedisConnection()
+    const redis = await getRedisConnection();
     await redis.mset(strings);
   } catch (e) {
     await sendMessage(

coins/src/scripts/coingecko.ts

@@ -21,6 +21,7 @@ import {
   fetchCgPriceData,
   retryCoingeckoRequest,
 } from "../utils/getCoinsUtils";
+import { storeAllTokens } from "../utils/shared/bridgedTvlPostgres";
 
 enum COIN_TYPES {
   over100m = "over100m",
@@ -141,6 +142,8 @@ async function getSymbolAndDecimals(
   }
 }
 
+const aggregatedPlatforms: string[] = [];
+
 async function getAndStoreCoins(coins: Coin[], rejected: Coin[]) {
   const coinData = await fetchCgPriceData(coins.map((c) => c.id));
   await storeCGCoinMetadatas(coinData);
@@ -299,6 +302,7 @@ async function getAndStoreCoins(coins: Coin[], rejected: Coin[]) {
           await ddb.put(item);
         },
         coinPlatformData,
+        aggregatedPlatforms,
       ),
     ),
   );
@@ -396,6 +400,7 @@ async function fetchCoingeckoData(
       requests.push(getAndStoreCoins(coins.slice(i, i + step), rejected));
     }
     await Promise.all(requests);
+    await storeAllTokens(aggregatedPlatforms);
   }
   clearInterval(timer);
   if (rejected.length > 0) {

coins/src/utils/coingeckoPlatforms.ts

@@ -43,6 +43,7 @@ export async function iterateOverPlatforms(
   coin: Coin,
   iterator: (PK: string) => Promise<void>,
   coinPlatformData: any,
+  aggregatedPlatforms: string[],
 ) {
   const platforms = coin.platforms as StringObject;
   for (const platform in platforms) {
@@ -52,6 +53,7 @@ export async function iterateOverPlatforms(
         if (chain === undefined) {
           continue;
         }
+        aggregatePlatforms(chain, platforms[platform]!, aggregatedPlatforms);
         const address =
           chain + ":" + lowercase(platforms[platform]!, chain).trim();
         const PK = `asset#${address}`;
@@ -101,3 +103,13 @@ export async function getCoinPlatformData(coins: Coin[]) {
   }
   return coinPlatformData;
 }
+
+export async function aggregatePlatforms(
+  chain: string,
+  address: string,
+  aggregatedPlatforms: string[],
+) {
+  const normalizedAddress =
+    chain in chainsThatShouldNotBeLowerCased ? address : address.toLowerCase();
+  aggregatedPlatforms.push(`${chain}:${normalizedAddress}`);
+}

defi/l2/adapters/thirdParty.ts

@@ -1,7 +1,8 @@
 import { Chain } from "@defillama/sdk/build/general";
 import providers from "@defillama/sdk/build/providers.json";
 import { Address } from "@defillama/sdk/build/types";
-import { allChainKeys, mixedCaseChains } from "../constants";
+import { allChainKeys } from "../constants";
+import { bridgedTvlMixedCaseChains } from "../../src/utils/shared/constants";
 import fetch from "node-fetch";
 import { additional, excluded } from "./manual";
 import axios from "axios";
@@ -22,6 +23,18 @@ Object.keys(providers).map((c: string) => {
   chainIdMap[providers[c as keyof typeof providers].chainId] = c;
 });
 
+const hyperlane = async (): Promise<void> => {
+  const bridge = "hyperlane";
+  if (!(bridge in bridgePromises))
+    bridgePromises[bridge] = fetch(
+      "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json"
+    ).then((r) => r.json());
+  const data = await bridgePromises[bridge];
+  if (doneAdapters.includes(bridge)) return;
+  data.map(({ address }: any) => addresses.eclipse.push(address));
+  doneAdapters.push(bridge);
+};
+
 const axelar = async (): Promise<void> => {
   const bridge = "axelar";
   if (!(bridge in bridgePromises))
@@ -98,7 +111,7 @@ const celer = async (): Promise<void> => {
   doneAdapters.push(bridge);
 };
 
-const adapters = [axelar(), wormhole(), celer()];
+const adapters = [axelar(), wormhole(), celer(), hyperlane()];
 const filteredAddresses: { [chain: Chain]: Address[] } = {};
 
 const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => {
@@ -108,12 +121,12 @@ const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => {
   Object.keys(addresses).map((chain: string) => {
     let chainAddresses =
       chain in excluded ? addresses[chain].filter((t: string) => !excluded[chain].includes(t)) : addresses[chain];
-    if (!mixedCaseChains.includes(chain)) chainAddresses = chainAddresses.map((t: string) => t.toLowerCase());
+    if (!bridgedTvlMixedCaseChains.includes(chain)) chainAddresses = chainAddresses.map((t: string) => t.toLowerCase());
     if (!(chain in additional)) {
       filteredAddresses[chain] = chainAddresses;
       return;
     }
-    const additionalTokens = mixedCaseChains.includes(chain)
+    const additionalTokens = bridgedTvlMixedCaseChains.includes(chain)
       ? additional[chain]
       : additional[chain].map((t: string) => t.toLowerCase());
     filteredAddresses[chain] = [...chainAddresses, ...additionalTokens];

defi/l2/cli/backfillSingle.ts

@@ -4,7 +4,7 @@ import findTvls from "../tvl";
 import { overwrite, parsePgData } from "../storeToDb";
 import setEnvSecrets from "../../src/utils/shared/setEnvSecrets";
 import postgres from "postgres";
-import { queryPostgresWithRetry } from "../layer2pg";
+import { queryPostgresWithRetry } from "../../src/utils/shared/bridgedTvlPostgres";
 import { ChartData } from "../types";
 
 // any mappings for chains in proc() needed??

defi/l2/constants.ts

@@ -8,7 +8,6 @@ export const excludedTvlKeys = ["PK", "SK", "tvl"];
 
 export const geckoSymbols = cgSymbols as { [key: string]: string };
 
-export const mixedCaseChains: string[] = ["solana", "tron", "sui", "aptos", "eclipse"];
 export const chainsWithoutCanonicalBridges: string[] = [
   "cronos",
   "kava",

defi/l2/native.ts

@@ -1,5 +1,5 @@
 import { getCurrentUnixTimestamp } from "../src/utils/date";
-import { fetchAllTokens } from "./layer2pg";
+import { fetchAllTokens } from "../src/utils/shared/bridgedTvlPostgres";
 import { McapData, TokenTvlData, DollarValues } from "./types";
 import { Chain } from "@defillama/sdk/build/general";
 import BigNumber from "bignumber.js";

defi/l2/storeToDb.ts

@@ -1,5 +1,5 @@
 import postgres from "postgres";
-import { queryPostgresWithRetry } from "../l2/layer2pg";
+import { queryPostgresWithRetry } from "../src/utils/shared/bridgedTvlPostgres";
 import { ChainTokens, ChartData, FinalChainData, FinalData } from "./types";
 import { getCurrentUnixTimestamp } from "../src/utils/date";
 import { getR2JSONString, storeR2JSONString } from "../src/utils/r2";

defi/l2/utils.ts

@@ -1,15 +1,16 @@
 import BigNumber from "bignumber.js";
 import { AllProtocols, CoinsApiData, McapsApiData, TokenTvlData } from "./types";
-import { canonicalBridgeIds, excludedTvlKeys, geckoSymbols, mixedCaseChains, zero } from "./constants";
+import { canonicalBridgeIds, excludedTvlKeys, geckoSymbols, zero } from "./constants";
 import fetch from "node-fetch";
+import { bridgedTvlMixedCaseChains } from "../src/utils/shared/constants";
 import sleep from "../src/utils/shared/sleep";
 import { call, multiCall } from "@defillama/sdk/build/abi/abi2";
 import { Address } from "@defillama/sdk/build/types";
 import * as incomingAssets from "./adapters";
 import { additional, excluded } from "./adapters/manual";
 import { Chain } from "@defillama/sdk/build/general";
 import PromisePool from "@supercharge/promise-pool";
-import { storeNotTokens } from "./layer2pg";
+import { storeNotTokens } from "../src/utils/shared/bridgedTvlPostgres";
 import { getBlock } from "@defillama/sdk/build/util/blocks";
 import { Connection, PublicKey } from "@solana/web3.js";
 import * as sdk from "@defillama/sdk";
@@ -325,7 +326,7 @@ async function getEVMSupplies(
         block: block.block,
       });
       contracts.slice(i, i + step).map((c: Address, i: number) => {
-        if (res[i]) supplies[`${chain}:${mixedCaseChains.includes(chain) ? c : c.toLowerCase()}`] = res[i];
+        if (res[i]) supplies[`${chain}:${bridgedTvlMixedCaseChains.includes(chain) ? c : c.toLowerCase()}`] = res[i];
       });
     } catch {
       try {
@@ -342,7 +343,8 @@ async function getEVMSupplies(
               await sleep(2000);
               if (chain == "tron") console.log(`${target}:: \t ${e.message}`);
             });
-            if (res) supplies[`${chain}:${mixedCaseChains.includes(chain) ? target : target.toLowerCase()}`] = res;
+            if (res)
+              supplies[`${chain}:${bridgedTvlMixedCaseChains.includes(chain) ? target : target.toLowerCase()}`] = res;
           });
       } catch (e) {
         if (chain == "tron") console.log(`tron supply call failed`);
@@ -381,11 +383,11 @@ export async function fetchBridgeTokenList(chain: Chain): Promise<Address[]> {
     tokens.push(...((await fetchThirdPartyTokenList())[chain] ?? []));
     let filteredTokens: Address[] =
       chain in excluded ? tokens.filter((t: string) => !excluded[chain].includes(t)) : tokens;
-    if (!mixedCaseChains.includes(chain)) filteredTokens = filteredTokens.map((t: string) => t.toLowerCase());
+    if (!bridgedTvlMixedCaseChains.includes(chain)) filteredTokens = filteredTokens.map((t: string) => t.toLowerCase());
 
     if (!(chain in additional)) return filteredTokens;
 
-    const additionalTokens = mixedCaseChains.includes(chain)
+    const additionalTokens = bridgedTvlMixedCaseChains.includes(chain)
       ? additional[chain]
       : additional[chain].map((t: string) => t.toLowerCase());
 

defi/src/cli/users/findMissingUsers.ts

@@ -4,7 +4,7 @@ import getRecordEarliestTimestamp from "../../utils/shared/getRecordEarliestTime
 import { dailyTvl } from "../../utils/getLastRecord";
 import { DAY } from "../../utils/date";
 import {date} from '../utils'
-import { getAccountsDBConnection } from "../../getDBConnection";
+import { getAccountsDBConnection } from "../../utils/shared/getDBConnection";
 
 async function main(){
     const table = [

defi/src/cli/users/findProtocolsMissingHistoricalData.ts

@@ -1,6 +1,6 @@
 require("dotenv").config();
 import { addressList } from "../../../dimension-adapters/users/list";
-import { getAccountsDBConnection } from "../../getDBConnection";
+import { getAccountsDBConnection } from "../../utils/shared/getDBConnection";
 import { DAY } from "../../utils/date";
 import { storeAllNewUsers } from "./queries/newUsers";
 

defi/src/cli/users/single/refillSingleProtocol.ts

@@ -1,5 +1,5 @@
 import { addressList } from "../../../../dimension-adapters/users/list";
-import { getAccountsDBConnection } from "../../../getDBConnection";
+import { getAccountsDBConnection } from "../../../utils/shared/getDBConnection";
 import { storeChainGas } from "../queries/gas";
 import { storeAllNewUsers } from "../queries/newUsers";
 import { storeChainTxs } from "../queries/txs";

defi/src/cli/users/single/utils/deleteUserData.ts

@@ -1,4 +1,4 @@
-import { getAccountsDBConnection } from "../../../../getDBConnection"
+import { getAccountsDBConnection } from "../../../../utils/shared/getDBConnection"
 
 export async function deleteUserDataForProtocol(protocolId:string){
     const sql = getAccountsDBConnection()

defi/src/reportError.ts

@@ -1,4 +1,4 @@
-import { getErrorDBConnection } from "./getDBConnection";
+import { getErrorDBConnection } from "./utils/shared/getDBConnection";
 import { getCurrentUnixTimestamp } from "./utils/date";
 import { sendMessage } from "./utils/discord";
 import { wrap, IResponse, successResponse, errorResponse } from "./utils/shared";