Merge branch 'min-conditions-merge' into 'main'

Min conditions applications, few fixes

See merge request flarenetwork/ftso/ftso-scaling!148

.gitignore

@@ -10,4 +10,6 @@ calculations
 test-db
 exports-csv
 staking-data
-listed-data-providers
+listed-data-providers
+passes-data
+rewards-data

apps/ftso-reward-calculation-process/src/libs/calculator-utils.ts

@@ -137,7 +137,7 @@ export async function calculationOfRewardCalculationDataForRange(
       );
       done = true;
     } catch (e) {
-      // console.log(e);
+      console.log(e);
       logger.error(
         `Error while calculating reward calculation data for voting rounds ${firstVotingRoundId}-${lastVotingRoundId} in reward epoch ${rewardEpochId}: ${e}`
       );

apps/ftso-reward-calculation-process/src/services/calculator.service.ts

@@ -4,13 +4,15 @@ import FakeTimers from "@sinonjs/fake-timers";
 import { EntityManager } from "typeorm";
 import { DataManagerForRewarding } from "../../../../libs/fsp-rewards/src/DataManagerForRewarding";
 import { IndexerClientForRewarding } from "../../../../libs/fsp-rewards/src/IndexerClientForRewarding";
-import { RewardEpochManager } from "../../../../libs/ftso-core/src/RewardEpochManager";
+import { BURN_ADDRESS, FUTURE_VOTING_ROUNDS } from "../../../../libs/fsp-rewards/src/constants";
+import { calculateMinimalConditions, extractNewPasses, updateClaimsForMinimalConditions } from "../../../../libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions";
+import { writeDataProviderConditions, writePassesInfo } from "../../../../libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions-data";
 import { initializeRewardEpochStorage } from "../../../../libs/fsp-rewards/src/reward-calculation/reward-calculation";
 import { RewardClaim } from "../../../../libs/fsp-rewards/src/utils/RewardClaim";
-import { RewardEpochDuration } from "../../../../libs/ftso-core/src/utils/RewardEpochDuration";
 import { deserializeAggregatedClaimsForVotingRoundId } from "../../../../libs/fsp-rewards/src/utils/stat-info/aggregated-claims";
 import { serializeFinalRewardClaims } from "../../../../libs/fsp-rewards/src/utils/stat-info/final-reward-claims";
 import { getIncrementalCalculationsTempRewards, serializeIncrementalCalculationsTempRewards } from "../../../../libs/fsp-rewards/src/utils/stat-info/incremental-calculation-temp-rewards";
+import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/fsp-rewards/src/utils/stat-info/incremental-calculation-temp-selected-feeds";
 import { recordProgress } from "../../../../libs/fsp-rewards/src/utils/stat-info/progress";
 import {
   RewardCalculationStatus,
@@ -25,8 +27,11 @@ import {
   serializeRewardEpochInfo,
 } from "../../../../libs/fsp-rewards/src/utils/stat-info/reward-epoch-info";
 import { destroyStorage } from "../../../../libs/fsp-rewards/src/utils/stat-info/storage";
+import { RewardEpochManager } from "../../../../libs/ftso-core/src/RewardEpochManager";
+import { RewardEpochDuration } from "../../../../libs/ftso-core/src/utils/RewardEpochDuration";
 import { IncrementalCalculationState } from "../interfaces/IncrementalCalculationState";
 import { OptionalCommandOptions } from "../interfaces/OptionalCommandOptions";
+import { calculateAttestationTypeAppearances } from "../libs/attestation-type-appearances";
 import {
   calculationOfRewardCalculationDataForRange,
   latestRewardEpochStart,
@@ -43,11 +48,6 @@ import { fullRoundOfferCalculation, initializeTemplateOffers } from "../libs/off
 import { runRandomNumberFixing } from "../libs/random-number-fixing-utils";
 import { runCalculateRewardClaimsTopJob } from "../libs/reward-claims-calculation";
 import { runCalculateRewardCalculationTopJob } from "../libs/reward-data-calculation";
-import { getIncrementalCalculationsFeedSelections, serializeIncrementalCalculationsFeedSelections } from "../../../../libs/fsp-rewards/src/utils/stat-info/incremental-calculation-temp-selected-feeds";
-import { calculateAttestationTypeAppearances } from "../libs/attestation-type-appearances";
-import { calculateMinimalConditions } from "../../../../libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions";
-import { writeDataProviderConditions } from "../../../../libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions-data";
-import {BURN_ADDRESS, FUTURE_VOTING_ROUNDS} from "../../../../libs/fsp-rewards/src/constants";
 
 if (process.env.FORCE_NOW) {
   const newNow = parseInt(process.env.FORCE_NOW) * 1000;
@@ -300,9 +300,14 @@ export class CalculatorService {
     if (options.rewardEpochId === undefined) {
       throw new Error("Reward epoch id is required for minimal conditions calculation");
     }
-    const result = calculateMinimalConditions(options.rewardEpochId, false);
+    const result = calculateMinimalConditions(options.rewardEpochId);
     writeDataProviderConditions(options.rewardEpochId, result);
+    const passes = extractNewPasses(result);
+    writePassesInfo(options.rewardEpochId, passes);
+    // conditional
+    updateClaimsForMinimalConditions(options.rewardEpochId, result);
   }
+
   /**
    * Returns a list of all (merged) reward claims for the given reward epoch.
    * Calculation can be quite intensive.

libs/contracts/src/constants.ts

@@ -69,7 +69,7 @@ export const FLARE_CONTRACTS: NetworkContractAddresses = {
         name: "FastUpdateIncentiveManager",
         address: "0xd648e8ACA486Ce876D641A0F53ED1F2E9eF4885D",
     },
-    FdcHub: {name: "FdcHub", address: ""},
+    FdcHub: {name: "FdcHub", address: "0xc25c749DC27Efb1864Cb3DADa8845B7687eB2d44"},
 };
 
 export type networks = "local-test" | "from-env" | "coston2" | "coston" | "songbird" | "flare";

libs/fsp-rewards/src/DataManagerForRewarding.ts

@@ -1,7 +1,4 @@
-import { ECDSASignature } from "../../ftso-core/src/fsp-utils/ECDSASignature";
-import { ProtocolMessageMerkleRoot } from "../../ftso-core/src/fsp-utils/ProtocolMessageMerkleRoot";
-import { RelayMessage } from "../../ftso-core/src/fsp-utils/RelayMessage";
-import { ISignaturePayload, SignaturePayload } from "../../ftso-core/src/fsp-utils/SignaturePayload";
+import { ContractMethodNames } from "../../contracts/src/definitions";
 import { DataAvailabilityStatus, DataManager, DataMangerResponse } from "../../ftso-core/src/DataManager";
 import {
   BlockAssuranceResult,
@@ -10,30 +7,31 @@ import {
   ParsedFinalizationData,
   SubmissionData,
 } from "../../ftso-core/src/IndexerClient";
-import { IndexerClientForRewarding } from "./IndexerClientForRewarding";
 import { RewardEpoch } from "../../ftso-core/src/RewardEpoch";
 import { RewardEpochManager } from "../../ftso-core/src/RewardEpochManager";
-import { ContractMethodNames } from "../../contracts/src/definitions";
 import {
-  ADDITIONAL_REWARDED_FINALIZATION_WINDOWS,
   EPOCH_SETTINGS,
-  FDC_PROTOCOL_ID,
   FTSO2_PROTOCOL_ID,
 } from "../../ftso-core/src/constants";
 import { DataForCalculations } from "../../ftso-core/src/data/DataForCalculations";
-import { bitVoteIndicesNum, extractFDCRewardData, uniqueRequestsIndices } from "./reward-calculation/fdc/fdc-utils";
+import { ECDSASignature } from "../../ftso-core/src/fsp-utils/ECDSASignature";
+import { ProtocolMessageMerkleRoot } from "../../ftso-core/src/fsp-utils/ProtocolMessageMerkleRoot";
+import { RelayMessage } from "../../ftso-core/src/fsp-utils/RelayMessage";
+import { ISignaturePayload, SignaturePayload } from "../../ftso-core/src/fsp-utils/SignaturePayload";
+import { SigningPolicy } from "../../ftso-core/src/fsp-utils/SigningPolicy";
 import { ILogger } from "../../ftso-core/src/utils/ILogger";
 import { errorString } from "../../ftso-core/src/utils/error";
 import { Address, MessageHash } from "../../ftso-core/src/voting-types";
-import { WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH } from "./constants";
+import { IndexerClientForRewarding } from "./IndexerClientForRewarding";
+import { ADDITIONAL_REWARDED_FINALIZATION_WINDOWS, FDC_PROTOCOL_ID, WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH } from "./constants";
 import {
   DataForRewardCalculation,
-  FastUpdatesDataForVotingRound,
   FDCDataForVotingRound,
   FDCRewardData,
+  FastUpdatesDataForVotingRound,
   PartialFDCDataForVotingRound,
 } from "./data-calculation-interfaces";
-import { SigningPolicy } from "../../ftso-core/src/fsp-utils/SigningPolicy";
+import { bitVoteIndicesNum, extractFDCRewardData, uniqueRequestsIndices } from "./reward-calculation/fdc/fdc-utils";
 
 export interface SignAndFinalizeSubmissionData {
   signatures: SubmissionData[];

libs/fsp-rewards/src/constants.ts

@@ -1,4 +1,74 @@
-import {networks} from "../../contracts/src/constants";
+import { networks } from "../../contracts/src/constants";
+
+export const ZERO_BYTES32 = "0x0000000000000000000000000000000000000000000000000000000000000000";
+
+// Protocol ids
+const ftso2FastUpdatesProtocolId = () => {
+    const network = process.env.NETWORK as networks;
+    switch (network) {
+        case "coston":
+        case "from-env":
+        case "local-test":
+        case "coston2":
+        case "songbird":
+        case "flare":
+            return 255;
+        default:
+            // Ensure exhaustive checking
+            // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
+            ((_: never): void => { })(network);
+    }
+};
+
+// Protocol id for FTSO2 fast updates
+export const FTSO2_FAST_UPDATES_PROTOCOL_ID = ftso2FastUpdatesProtocolId();
+
+
+const FDCProtocolId = () => {
+    const network = process.env.NETWORK as networks;
+    switch (network) {
+        case "coston":
+        case "from-env":
+        case "local-test":
+        case "coston2":
+        case "songbird":
+        case "flare":
+            return 200;
+        default:
+            // Ensure exhaustive checking
+            // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
+            ((_: never): void => { })(network);
+    }
+};
+
+// Protocol id for FDC
+export const FDC_PROTOCOL_ID = FDCProtocolId();
+
+export const STAKING_PROTOCOL_ID = 0;
+
+/**
+ * The number of additional voting rounds for performing queries for signature and finalization data.
+ * If value is 0, then for votingRoundId the original window is from the end of reveals to the end
+ * of the voting epoch votingRoundId. If value is bigger, it extends to ends of the next epochs accordingly.
+ */
+const additionalRewardFinalizationWindows = () => {
+    const network = process.env.NETWORK as networks;
+    switch (network) {
+        case "from-env":
+        case "coston":
+        case "coston2":
+        case "songbird":
+        case "flare":
+        case "local-test":
+            return 0;
+        default:
+            // Ensure exhaustive checking
+            // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
+            ((_: never): void => { })(network);
+    }
+};
+
+export const ADDITIONAL_REWARDED_FINALIZATION_WINDOWS = additionalRewardFinalizationWindows();
 
 const burnAddress = () => {
     const network = process.env.NETWORK as networks;

libs/fsp-rewards/src/reward-calculation/fdc/fdc-utils.ts

@@ -1,10 +1,9 @@
-import { ISignaturePayload } from "../../../../ftso-core/src/fsp-utils/SignaturePayload";
+import { AttestationRequest } from "../../../../contracts/src/events/AttestationRequest";
 import { GenericSubmissionData, SubmissionData } from "../../../../ftso-core/src/IndexerClient";
 import { RewardEpoch } from "../../../../ftso-core/src/RewardEpoch";
-import { FDC_PROTOCOL_ID } from "../../../../ftso-core/src/constants";
-import { AttestationRequest } from "../../../../contracts/src/events/AttestationRequest";
+import { ISignaturePayload } from "../../../../ftso-core/src/fsp-utils/SignaturePayload";
 import { Address, MessageHash } from "../../../../ftso-core/src/voting-types";
-import {WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH} from "../../constants";
+import { FDC_PROTOCOL_ID, WRONG_SIGNATURE_INDICATOR_MESSAGE_HASH } from "../../constants";
 import {
    FDCEligibleSigner,
    FDCOffender,
@@ -22,7 +21,7 @@ export function uniqueRequestsIndices(attestationRequests: AttestationRequest[])
    for (let i = 0; i < attestationRequests.length; i++) {
       const request = attestationRequests[i];
       if (!encountered.get(request.data)) {
-         encountered.set(request.data, i);
+         encountered.set(request.data, result.length);
          result.push([i]);
       } else {
          result[encountered.get(request.data)].push(i);

libs/fsp-rewards/src/reward-calculation/fdc/reward-fdc-penalties.ts

@@ -1,12 +1,12 @@
 import { VoterWeights } from "../../../../ftso-core/src/RewardEpoch";
-import { FDC_PROTOCOL_ID } from "../../../../ftso-core/src/constants";
 import { IPartialRewardOfferForRound } from "../../utils/PartialRewardOffer";
 import { IPartialRewardClaim } from "../../utils/RewardClaim";
 import { SDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data";
 import { RewardEpochInfo } from "../../utils/stat-info/reward-epoch-info";
 import { Address } from "../../../../ftso-core/src/voting-types";
 import { RewardTypePrefix } from "../RewardTypePrefix";
 import { generateSigningWeightBasedClaimsForVoter } from "../reward-signing-split";
+import { FDC_PROTOCOL_ID } from "../../constants";
 
 /**
  *  * Given a @param offer, @param penaltyFactor and @param votersWeights penalty claims for offenders.

libs/fsp-rewards/src/reward-calculation/fdc/reward-fdc-signing.ts

@@ -1,15 +1,15 @@
-import { EPOCH_SETTINGS, FDC_PROTOCOL_ID } from "../../../../ftso-core/src/constants";
+import { EPOCH_SETTINGS } from "../../../../ftso-core/src/constants";
+import { Address } from "../../../../ftso-core/src/voting-types";
+import { FDC_PROTOCOL_ID, FINALIZATION_BIPS, TOTAL_BIPS } from "../../constants";
+import { FDCEligibleSigner } from "../../data-calculation-interfaces";
 import { IPartialRewardOfferForRound } from "../../utils/PartialRewardOffer";
 import { ClaimType, IPartialRewardClaim } from "../../utils/RewardClaim";
 import { SDataForRewardCalculation } from "../../utils/stat-info/reward-calculation-data";
 import { RewardEpochInfo } from "../../utils/stat-info/reward-epoch-info";
-import { Address } from "../../../../ftso-core/src/voting-types";
 import { RewardTypePrefix } from "../RewardTypePrefix";
 import { SigningRewardClaimType } from "../reward-signing";
 import { generateSigningWeightBasedClaimsForVoter } from "../reward-signing-split";
 import { isSignatureBeforeTimestamp, isSignatureInGracePeriod } from "../reward-utils";
-import {FINALIZATION_BIPS, TOTAL_BIPS} from "../../constants";
-import {FDCEligibleSigner} from "../../data-calculation-interfaces";
 
 /**
  * A split of partial reward offer into three parts:

libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions-constants.ts

@@ -12,7 +12,7 @@
 export const TOTAL_PPM = 1000000n;
 export const FTSO_SCALING_AVAILABILITY_THRESHOLD_PPM = 800000n;      // 80%
 export const FTSO_SCALING_CLOSENESS_THRESHOLD_PPM = 5000n;           // 0.5%
-export const FU_THRESHOLD_PPM = 800000n;                             // 60%
+export const FU_THRESHOLD_PPM = 800000n;                             // 80%
 export const FU_CONSIDERATION_THRESHOLD_PPM = 2000n;                 // 0.2% of the weight
 export const STAKING_UPTIME_THRESHOLD_PPM = 800000n;                 // 80%
 export const STAKING_MIN_SELF_BOND_GWEI = 1000000000000000n;         // 1M FLR

libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions-data.ts

@@ -1,9 +1,10 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { base58 } from '@scure/base';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import path from "path/posix";
+import { bigIntReplacer, bigIntReviver } from "../../../../ftso-core/src/utils/big-number-serialization";
+import { CALCULATIONS_FOLDER, STAKING_DATA_FOLDER } from "../../constants";
+import { MINIMAL_CONDITIONS_FILE, PASSES_FILE } from '../../utils/stat-info/constants';
 import { DataProviderConditions, DataProviderPasses, ListedProviderList, ValidatorInfo } from "./minimal-conditions-interfaces";
-import { bigIntReplacer } from "../../../../ftso-core/src/utils/big-number-serialization";
-import {CALCULATIONS_FOLDER, PASSES_DATA_FOLDER, STAKING_DATA_FOLDER} from "../../constants";
 
 /**
  * Reads the staking info for a given reward epoch id. 
@@ -15,7 +16,7 @@ export function readStakingInfo(
 ): ValidatorInfo[] {
    const fname = path.join(stakingDataFolder, `${rewardEpochId}-nodes-data.json`);
    const data = readFileSync(fname, 'utf8');
-   const result: ValidatorInfo[] = JSON.parse(data);
+   const result: ValidatorInfo[] = JSON.parse(data, bigIntReviver);
    for(let validatorInfo of result) {
       // "NodeID-2a7BPY7UeJv2njMuyUHfBSTeQCYZj6bwV"
       // Checksum is not validated
@@ -30,29 +31,29 @@ export function readStakingInfo(
  */
 export function readPassesInfo(
    rewardEpochId: number,
-   passesDataFolder = PASSES_DATA_FOLDER()
-): DataProviderPasses[] | undefined {
-   const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`);
+   calculationFolder = CALCULATIONS_FOLDER()
+): DataProviderPasses[] {
+   const fname = path.join(calculationFolder, `${rewardEpochId}`, PASSES_FILE);
    if(!existsSync(fname)) {
-      return undefined;
+      throw new Error(`Passes file not found: ${fname}`);
    }
    const data = readFileSync(fname, 'utf8');
-   return JSON.parse(data);
+   return JSON.parse(data) as DataProviderPasses[];
 }
 
 /**
  * Writes the staking info for a given reward epoch id.
  */
 export function writePassesInfo(
    rewardEpochId: number,
-   data: DataProviderPasses,
-   passesDataFolder = PASSES_DATA_FOLDER()
-): void {
-   if (!existsSync(passesDataFolder)) {
-      mkdirSync(passesDataFolder, { recursive: true });
+   data: DataProviderPasses[],
+   calculationFolder = CALCULATIONS_FOLDER()
+): void {   
+   if (!existsSync(calculationFolder)) {
+      mkdirSync(calculationFolder, { recursive: true });
    }
-   const fname = path.join(passesDataFolder, `${rewardEpochId}-passes-data.json`);
-   writeFileSync(fname, JSON.stringify(data));
+   const fname = path.join(calculationFolder, `${rewardEpochId}`, PASSES_FILE);
+   writeFileSync(fname, JSON.stringify(data, null, 2));
 }
 
 /**
@@ -66,7 +67,7 @@ export function writeDataProviderConditions(
    if (!existsSync(calculationFolder)) {
       mkdirSync(calculationFolder, { recursive: true });
    }
-   const fname = path.join(calculationFolder, `${rewardEpochId}`, `minimal-conditions.json`);
+   const fname = path.join(calculationFolder, `${rewardEpochId}`, MINIMAL_CONDITIONS_FILE);
    writeFileSync(fname, JSON.stringify(data, bigIntReplacer, 2));
 }
 

libs/fsp-rewards/src/reward-calculation/minimal-conditions/minimal-conditions-interfaces.ts

@@ -31,8 +31,8 @@ export interface ValidatorInfo {
    fee: number;
    // group number
    group: number;
-   // is the validator eligible for staking rewards
-   eligible: boolean;
+   // is the validator for uptime
+   uptimeEligible: boolean;
    // data provider name
    ftsoName: string;
    // Boosting eligibility bond in GWei
@@ -95,9 +95,13 @@ export interface MinimalConditionFailure {
 
 export interface DataProviderPasses {
    // epoch id in string
-   rewardEpochId: string;
+   rewardEpochId: number;
    // voter identity address in lowercase
    voterAddress: string;
+   // data provider name
+   dataProviderName?: string;
+   // if the voter was eligible for reward in the reward epoch
+   eligibleForReward: boolean;
    // number of passes. A number between 0 and 3
    passes: number;
    // failures
@@ -118,6 +122,10 @@ export interface DataProviderConditions {
    dataProviderName?: string;
    // voter identity address
    voterAddress: string;
+   // voter delegation address
+   delegationAddress: string;
+   // list of node ids assigned to voter
+   nodeIds: string[];
    // voter index
    voterIndex: number;
    // passes held before the reward epoch calculation