Merge remote-tracking branch 'origin/dev' into feat/checker-integration

defi-wonderland · Jan 14, 2025 · 46422a5 · 46422a5
2 parents 376e3b4 + 1f95d53
commit 46422a5
Show file tree

Hide file tree

Showing 116 changed files with 2,890 additions and 756 deletions.
diff --git a/apps/processing/.env.example b/apps/processing/.env.example
@@ -13,4 +13,9 @@ IPFS_GATEWAYS_URL=["https://ipfs.io","https://gateway.pinata.cloud","https://dwe
 PRICING_SOURCE= # 'coingecko' or 'dummy'
 
 COINGECKO_API_KEY={{YOUR_KEY}}
-COINGECKO_API_TYPE=demo
+COINGECKO_API_TYPE=demo
+
+RETRY_MAX_ATTEMPTS=3
+RETRY_BASE_DELAY_MS=3000
+RETRY_FACTOR=2
+RETRY_MAX_DELAY_MS=300000
diff --git a/apps/processing/README.md b/apps/processing/README.md
@@ -36,6 +36,10 @@ Available options:
 | `DUMMY_PRICE` | Dummy price | 1 | No | Only if PRICING_SOURCE is dummy |
 | `COINGECKO_API_KEY` | API key for CoinGecko service | N/A | Yes | |
 | `COINGECKO_API_TYPE` | CoinGecko API tier (demo or pro) | pro | No | |
+| `RETRY_MAX_ATTEMPTS` | Maximum number of retry attempts | 3 | No | |
+| `RETRY_BASE_DELAY_MS` | Base delay for retry attempts | 3000 | No | |
+| `RETRY_FACTOR` | Delay factor for retry attempts | 2 | No | |
+| `RETRY_MAX_DELAY_MS` | Maximum delay for retry attempts | 300000 | No | |
 
 ## Available Scripts
 

diff --git a/apps/processing/src/config/env.ts b/apps/processing/src/config/env.ts
@@ -36,6 +36,10 @@ const baseSchema = z.object({
     IPFS_GATEWAYS_URL: stringToJSONSchema
         .pipe(z.array(z.string().url()))
         .default('["https://ipfs.io"]'),
+    RETRY_MAX_ATTEMPTS: z.coerce.number().int().min(1).default(3),
+    RETRY_BASE_DELAY_MS: z.coerce.number().int().min(1).default(3000), // 3 seconds
+    RETRY_FACTOR: z.coerce.number().int().min(1).default(2),
+    RETRY_MAX_DELAY_MS: z.coerce.number().int().min(1).optional(), // 5 minute
 });
 
 const dummyPricingSchema = baseSchema.extend({

diff --git a/apps/processing/src/services/processing.service.ts b/apps/processing/src/services/processing.service.ts
@@ -45,8 +45,14 @@ export class ProcessingService {
     static async initialize(env: Environment): Promise<ProcessingService> {
         const sharedDependencies = await SharedDependenciesService.initialize(env);
         const { CHAINS: chains } = env;
-        const { core, registriesRepositories, indexerClient, kyselyDatabase, logger } =
-            sharedDependencies;
+        const {
+            core,
+            registriesRepositories,
+            indexerClient,
+            kyselyDatabase,
+            logger,
+            retryStrategy,
+        } = sharedDependencies;
         const {
             eventRegistryRepository,
             strategyRegistryRepository,
@@ -83,6 +89,7 @@ export class ProcessingService {
                 },
                 chain.fetchLimit,
                 chain.fetchDelayMs,
+                retryStrategy,
                 logger,
             );
             const retroactiveProcessor = new RetroactiveProcessor(
@@ -95,6 +102,7 @@ export class ProcessingService {
                     checkpointRepository: strategyProcessingCheckpointRepository,
                 },
                 chain.fetchLimit,
+                retryStrategy,
                 logger,
             );
 

diff --git a/apps/processing/src/services/sharedDependencies.service.ts b/apps/processing/src/services/sharedDependencies.service.ts
@@ -1,7 +1,7 @@
 import { CoreDependencies } from "@grants-stack-indexer/data-flow";
 import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client";
-import { IpfsProvider } from "@grants-stack-indexer/metadata";
-import { PricingProviderFactory } from "@grants-stack-indexer/pricing";
+import { CachingMetadataProvider, IpfsProvider } from "@grants-stack-indexer/metadata";
+import { CachingPricingProvider, PricingProviderFactory } from "@grants-stack-indexer/pricing";
 import {
     createKyselyDatabase,
     IEventRegistryRepository,
@@ -11,12 +11,15 @@ import {
     KyselyApplicationRepository,
     KyselyDonationRepository,
     KyselyEventRegistryRepository,
+    KyselyMetadataCache,
+    KyselyPricingCache,
     KyselyProjectRepository,
     KyselyRoundRepository,
     KyselyStrategyProcessingCheckpointRepository,
     KyselyStrategyRegistryRepository,
+    KyselyTransactionManager,
 } from "@grants-stack-indexer/repository";
-import { ILogger, Logger } from "@grants-stack-indexer/shared";
+import { ExponentialBackoff, ILogger, Logger, RetryStrategy } from "@grants-stack-indexer/shared";
 
 import { Environment } from "../config/index.js";
 
@@ -29,6 +32,7 @@ export type SharedDependencies = {
     };
     indexerClient: EnvioIndexerClient;
     kyselyDatabase: ReturnType<typeof createKyselyDatabase>;
+    retryStrategy: RetryStrategy;
     logger: ILogger;
 };
 
@@ -50,6 +54,8 @@ export class SharedDependenciesService {
             logger,
         );
 
+        const transactionManager = new KyselyTransactionManager(kyselyDatabase);
+
         const projectRepository = new KyselyProjectRepository(kyselyDatabase, env.DATABASE_SCHEMA);
         const roundRepository = new KyselyRoundRepository(kyselyDatabase, env.DATABASE_SCHEMA);
         const applicationRepository = new KyselyApplicationRepository(
@@ -64,11 +70,23 @@ export class SharedDependenciesService {
             kyselyDatabase,
             env.DATABASE_SCHEMA,
         );
+        const pricingRepository = new KyselyPricingCache(kyselyDatabase, env.DATABASE_SCHEMA);
         const pricingProvider = PricingProviderFactory.create(env, {
             logger,
         });
+        const cachedPricingProvider = new CachingPricingProvider(
+            pricingProvider,
+            pricingRepository,
+            logger,
+        );
 
+        const metadataRepository = new KyselyMetadataCache(kyselyDatabase, env.DATABASE_SCHEMA);
         const metadataProvider = new IpfsProvider(env.IPFS_GATEWAYS_URL, logger);
+        const cachedMetadataProvider = new CachingMetadataProvider(
+            metadataProvider,
+            metadataRepository,
+            logger,
+        );
 
         const eventRegistryRepository = new KyselyEventRegistryRepository(
             kyselyDatabase,
@@ -88,15 +106,23 @@ export class SharedDependenciesService {
             env.INDEXER_ADMIN_SECRET,
         );
 
+        const retryStrategy = new ExponentialBackoff({
+            maxAttempts: env.RETRY_MAX_ATTEMPTS,
+            baseDelay: env.RETRY_BASE_DELAY_MS,
+            maxDelay: env.RETRY_MAX_DELAY_MS,
+            factor: env.RETRY_FACTOR,
+        });
+
         return {
             core: {
                 projectRepository,
                 roundRepository,
                 applicationRepository,
-                pricingProvider,
+                pricingProvider: cachedPricingProvider,
                 donationRepository,
-                metadataProvider,
+                metadataProvider: cachedMetadataProvider,
                 applicationPayoutRepository,
+                transactionManager,
             },
             registriesRepositories: {
                 eventRegistryRepository,
@@ -105,6 +131,7 @@ export class SharedDependenciesService {
             },
             indexerClient,
             kyselyDatabase,
+            retryStrategy,
             logger,
         };
     }

diff --git a/apps/processing/test/unit/sharedDependencies.service.spec.ts b/apps/processing/test/unit/sharedDependencies.service.spec.ts
@@ -19,8 +19,10 @@ const mocks = vi.hoisted(() => {
     };
 });
 
-vi.mock("@grants-stack-indexer/shared", () => {
+vi.mock("@grants-stack-indexer/shared", async (importActual) => {
+    const actual = await importActual<typeof import("@grants-stack-indexer/shared")>();
     return {
+        ...actual,
         Logger: {
             getInstance: vi.fn().mockReturnValue(mocks.logger),
         },
@@ -42,16 +44,21 @@ vi.mock("@grants-stack-indexer/repository", () => ({
     })),
     KyselyEventRegistryRepository: vi.fn(),
     KyselyStrategyProcessingCheckpointRepository: vi.fn(),
+    KyselyTransactionManager: vi.fn(),
+    KyselyPricingCache: vi.fn(),
+    KyselyMetadataCache: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/pricing", () => ({
     PricingProviderFactory: {
         create: vi.fn(),
     },
+    CachingPricingProvider: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/metadata", () => ({
     IpfsProvider: vi.fn(),
+    CachingMetadataProvider: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/indexer-client", () => ({
@@ -145,6 +152,7 @@ describe("SharedDependenciesService", () => {
         expect(dependencies.core).toHaveProperty("donationRepository");
         expect(dependencies.core).toHaveProperty("metadataProvider");
         expect(dependencies.core).toHaveProperty("applicationPayoutRepository");
+        expect(dependencies.core).toHaveProperty("transactionManager");
 
         // Verify registries
         expect(dependencies.registriesRepositories).toHaveProperty("eventRegistryRepository");

diff --git a/packages/chain-providers/src/exceptions/dataDecode.exception.ts b/packages/chain-providers/src/exceptions/dataDecode.exception.ts
@@ -1,6 +1,7 @@
-export class DataDecodeException extends Error {
+import { NonRetriableError } from "@grants-stack-indexer/shared";
+
+export class DataDecodeException extends NonRetriableError {
     constructor(message: string) {
         super(message);
-        this.name = "DataDecodeException";
     }
 }
diff --git a/packages/chain-providers/src/exceptions/invalidArgument.exception.ts b/packages/chain-providers/src/exceptions/invalidArgument.exception.ts
@@ -1,6 +1,7 @@
-export class InvalidArgumentException extends Error {
+import { NonRetriableError } from "@grants-stack-indexer/shared";
+
+export class InvalidArgumentException extends NonRetriableError {
     constructor(message: string) {
         super(message);
-        this.name = "InvalidArgumentException";
     }
 }
diff --git a/packages/chain-providers/src/exceptions/multicallNotFound.exception.ts b/packages/chain-providers/src/exceptions/multicallNotFound.exception.ts
@@ -1,4 +1,6 @@
-export class MulticallNotFound extends Error {
+import { NonRetriableError } from "@grants-stack-indexer/shared";
+
+export class MulticallNotFound extends NonRetriableError {
     constructor() {
         super("Multicall contract address not found");
     }

diff --git a/packages/chain-providers/src/exceptions/rpcUrlsEmpty.exception.ts b/packages/chain-providers/src/exceptions/rpcUrlsEmpty.exception.ts
@@ -1,6 +1,7 @@
-export class RpcUrlsEmpty extends Error {
+import { NonRetriableError } from "@grants-stack-indexer/shared";
+
+export class RpcUrlsEmpty extends NonRetriableError {
     constructor() {
         super("RPC URLs array cannot be empty");
-        this.name = "RpcUrlsEmpty";
     }
 }