From 78ec428accf82e7e55c13e331e65e20edf6cf64a Mon Sep 17 00:00:00 2001 From: Gustavo Rivero Date: Mon, 23 Aug 2021 16:51:53 -0700 Subject: [PATCH] Merge branch 'main' into feature/W-9259597__ccsPKBandingPOC # Conflicts: # cumulusci.yml # force-app/main/default/classes/UTIL_pkBanding.cls # force-app/main/default/classes/UTIL_pkBanding.cls-meta.xml # force-app/main/default/classes/pkBandingService.cls # force-app/main/default/classes/pkBandingService.cls-meta.xml # force-app/main/default/classes/pkBanding_CTRL.cls # force-app/main/default/classes/pkBanding_CTRL.cls-meta.xml # force-app/main/default/pages/pkBanding.page # force-app/main/default/pages/pkBanding.page-meta.xml # force-app/main/default/pages/pkBandingAWS.page # force-app/main/default/pages/pkBandingAWS.page-meta.xml # src/classes/RD2_OpportunityEvaluationService.cls # src/package.xml --- .../default/classes/RD2_DatabaseService.cls | 39 +- .../RD2_OpportunityEvaluationService.cls | 16 + .../main/default/classes/UTIL_pkBanding.cls | 179 +++++ .../classes/UTIL_pkBanding.cls-meta.xml | 5 + .../main/default/classes/pkBandingService.cls | 254 +++++++ .../classes/pkBandingService.cls-meta.xml | 5 + .../main/default/classes/pkBanding_CTRL.cls | 27 + .../classes/pkBanding_CTRL.cls-meta.xml | 5 + force-app/main/default/pages/pkBanding.page | 712 ++++++++++++++++++ .../default/pages/pkBanding.page-meta.xml | 5 + 10 files changed, 1245 insertions(+), 2 deletions(-) create mode 100644 force-app/main/default/classes/UTIL_pkBanding.cls create mode 100644 force-app/main/default/classes/UTIL_pkBanding.cls-meta.xml create mode 100644 force-app/main/default/classes/pkBandingService.cls create mode 100644 force-app/main/default/classes/pkBandingService.cls-meta.xml create mode 100644 force-app/main/default/classes/pkBanding_CTRL.cls create mode 100644 force-app/main/default/classes/pkBanding_CTRL.cls-meta.xml create mode 100644 force-app/main/default/pages/pkBanding.page create mode 100644 force-app/main/default/pages/pkBanding.page-meta.xml diff --git a/force-app/main/default/classes/RD2_DatabaseService.cls b/force-app/main/default/classes/RD2_DatabaseService.cls index 916bef3a792..54caf52d1df 100644 --- a/force-app/main/default/classes/RD2_DatabaseService.cls +++ b/force-app/main/default/classes/RD2_DatabaseService.cls @@ -40,6 +40,7 @@ public without sharing class RD2_DatabaseService { private ERR_Handler.Errors errorResult = new ERR_Handler.Errors(); private Set failedRDIds = new Set(); + private Map failedRDIdsMap = new Map(); /*** * @description Creates specified records @@ -135,14 +136,31 @@ public without sharing class RD2_DatabaseService { : ((Database.DeleteResult)dmlResults[i]).isSuccess(); if (!isSuccess) { + List lErr = dmlResults[i] instanceof Database.SaveResult + ? ((Database.SaveResult)dmlResults[i]).getErrors() + : ((Database.DeleteResult)dmlResults[i]).getErrors(); + String sErr; + // Operation failed, so get all errors + for (Database.Error err : lErr) { + sErr = String.isBlank(sErr) + ? err.getMessage() + : sErr + ', ' + err.getMessage(); + } + failedRDIds.add((Id) records[i].get('npe03__Recurring_Donation__c')); + if (records[i].get('npe03__Recurring_Donation__c')!=null) { + failedRDIdsMap.put((Id) records[i].get('npe03__Recurring_Donation__c'), sErr); + } } } } else if (recordSObjectType == rdSObjectType) { - for (Error__c error : dmlErrors.errorRecords) { + for (Error__c error : dmlErrors.errorRecords) { failedRDIds.add(error.Related_Record_ID__c); - } + if (error.Related_Record_ID__c!=null) { + failedRDIdsMap.put(error.Related_Record_ID__c, error.Full_Message__c); + } + } } } @@ -166,4 +184,21 @@ public without sharing class RD2_DatabaseService { return failedRDIds.size(); } + /*** + * @description Returns Ids of failed Recurring Donations + * @return Set + */ + public Set getRecordsFailedId() { + failedRDIds.remove(null); + return failedRDIds; + } + + /*** + * @description Returns Map of failed Recurring Donations Id to error message + * @return Map + */ + public Map getRecordsFailedMap() { + return failedRDIdsMap; + } + } diff --git a/force-app/main/default/classes/RD2_OpportunityEvaluationService.cls b/force-app/main/default/classes/RD2_OpportunityEvaluationService.cls index 40ecd5210a3..f4a91da5954 100644 --- a/force-app/main/default/classes/RD2_OpportunityEvaluationService.cls +++ b/force-app/main/default/classes/RD2_OpportunityEvaluationService.cls @@ -303,6 +303,22 @@ public inherited sharing class RD2_OpportunityEvaluationService { return dbService.getRecordsFailed(); } + /** + * @description Returns set of Ids with failed RDs + * @return Set + */ + public Set getRecordsFailedId() { + return dbService.getRecordsFailedId(); + } + + /** + * @description Returns Map of Ids and error messages for failed RDs + * @return Map + */ + public Map getRecordsFailedMap() { + return dbService.getRecordsFailedMap(); + } + /** * @description Returns true if any of the key fields used to create or manage installment * Opportunities has been changed. diff --git a/force-app/main/default/classes/UTIL_pkBanding.cls b/force-app/main/default/classes/UTIL_pkBanding.cls new file mode 100644 index 00000000000..32a8b6f01ce --- /dev/null +++ b/force-app/main/default/classes/UTIL_pkBanding.cls @@ -0,0 +1,179 @@ +public with sharing class UTIL_pkBanding { + + public String querySelect = getQuery(); + public String queryWhere; + public pkBandingService.dispatcherResponse resp = new pkBandingService.dispatcherResponse(); + + // should be interface + public String getQuery() { + return new UTIL_Query() + .withFrom(npe03__Recurring_Donation__c.SObjectType) + .withSelectFields( + new Set{ 'Id' } + ) + .build(); + } + + // should be interface + public sObject[] getNextBatch(){ + SObject[] records = new List(); + resp.completed = false; + + while (records.isEmpty() && !resp.completed){ + records = queryNextBatch(); + if (records.isEmpty()){ + getStartOfNextCluster(); + } + } + return records; + } + + public SObject[] queryNextBatch(){ + if (resp.cursor.offset == null){ + resp.cursor.offset = getStartOfNextCluster(); + resp.cursor.offset = idAdd(resp.cursor.offset, -1); + } + if (getBand(resp.cursor.offset, resp.cursor.partitionBits) != resp.cursor.band){ + resp.cursor.offset = getStartOfNextChunkInBand( + resp.cursor.offset, + resp.cursor.band, + resp.cursor.partitionBits); + resp.cursor.offset = idAdd(resp.cursor.offset, -1); + } + String startOffset = resp.cursor.offset; + String endOfChunk = getNearestEndOfChunkInBand( + resp.cursor.offset, + resp.cursor.band, + resp.cursor.partitionBits); + Integer batchSize = resp.cursor.batchSize; + + String nextBatchQuery = querySelect + + (String.isblank(queryWhere)?' ':' ' + queryWhere) + + (String.isblank(queryWhere)?' WHERE ':' AND ') + + ' Id > \'' + startOffset + '\' AND '+ + ' Id <= \'' + endOfChunk + '\'' + + ' ORDER BY Id ASC LIMIT '+batchSize; + + System.debug('nextBatchQuery: ' + nextBatchQuery); + + SObject[] sObjectList = Database.query(nextBatchQuery); + + if (sObjectList.isEmpty()) { + resp.cursor.sparse = true; + } + if (sObjectList.size() :offset ':' ') + + ' ORDER BY Id ASC LIMIT 1'; + //System.assert(false, nextIdQuery); + sObject[] firstIdList = Database.query(nextIdQuery); + String minId; + If (!firstIdList.isEmpty()) { + minId = firstIdList[0].Id.to15(); + resp.cursor.offset = minId; + resp.cursor.sparse = false; + } else { + resp.completed = true; + } + return minId; + } + + + public String getStartOfNextChunkInBand(String b62, Integer band, Integer bandBits) { + //TO DO: What if band is not 3 bits? + System.assert(bandBits == 3, 'Only 3-bit bands are currently suported'); + System.assert(b62.length()==15,'b62 number must be exactly 15 chars long'); + String IdPrefix = b62.left(6); + String chunkPrefix = b62.mid(6,6); + //TO DO: figure out if I need to add offset 1 or not based on actual band in key + Integer nextBandOffset = getBand(b62, bandBits)>=band?1:0; + //System.assert(false,'chunkprefix: '+chunkPrefix + 'nextBandOffset: '+ nextBandOffset); + chunkPrefix = longToB62(b62toLong(chunkPrefix)+nextBandOffset,6); + String bandStr = longToB62(band<<3,1); + b62 = IdPrefix + chunkPrefix + bandStr + '00'; + return b62; + } + + + public String getNearestEndOfChunkInBand(String b62, Integer band, Integer bandBits) { + //TO DO: What if band is not 3 bits? + System.assert(bandBits == 3, 'Only 3-bit bands are currently suported'); + System.assert(b62.length()==15,'b62 number must be exactly 15 chars long'); + //b62 = getStartOfNextChunkInBand(b62, band, bandBits); + String IdPrefix = b62.left(6); + String chunkPrefix = b62.mid(6,6); + Integer nextBandOffset = getBand(b62, bandBits)>band?1:0; + chunkPrefix = longToB62(b62toLong(chunkPrefix)+nextBandOffset,6); + //TO DO: Cannot add 7 to band 111b or it will overflow!!! Subtract 10b? + Integer last3bits = band==7?5:7; + String bandStr = longToB62((band<<3)+last3bits,1); + b62 = IdPrefix + chunkPrefix + bandStr + 'zz'; + return b62; + } + + + public Integer getBand(String b62, Integer bandBits){ + System.assert(bandBits == 3, 'Only 3-bit bands are currently suported'); + System.assert(b62.length()==15,'b62 number must be exactly 15 chars long'); + Integer band; + band = (Integer)b62toLong(b62.mid(12,1)); + band >>>=3; + return band; + } + + + public String idAdd(String b62, Long addend){ + System.assert(b62.length()==15,'b62 number must be exactly 15 chars long'); + String IdPrefix = b62.left(6); + String sequenceB62 = b62.mid(6,9); + Long sequenceDec = b62toLong(sequenceB62); + sequenceDec += addend; + System.assert(sequenceDec>=0,'Negative Ids are not valid'); + sequenceB62 = longToB62(sequenceDec, 9); + b62 = IdPrefix + sequenceB62; + return b62; + } + + + public Long b62toLong(String b62){ + System.assert(b62.length()<=10,'b62 string is too long'); + String chars ='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; //[0-9A-Za-z] + Long sum=0; + for (String i: b62.split('')){ + sum = sum*62; + sum += chars.indexOf(i); + } + return sum; + } + + + public String longToB62(Long lng, Integer length) { + System.assert(lng >= 0, 'Cannot convert negative number to base62'); + System.assert(length <= 10, 'Largest supported base62 length is 10'); + String chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; //[0-9A-Za-z] + String b62 = ''; + while (lng > 0) { + Integer lchar = (Integer) Math.mod(lng, 62); + b62 = chars.mid(lchar, 1) + b62; + lng = lng / 62; + } + b62 = b62.leftPad(length, '0'); + System.assert(b62.length() <= length, 'b62 conversion OVERFLOW: ' + b62); + return b62; + } + +} \ No newline at end of file diff --git a/force-app/main/default/classes/UTIL_pkBanding.cls-meta.xml b/force-app/main/default/classes/UTIL_pkBanding.cls-meta.xml new file mode 100644 index 00000000000..dd61d1f917e --- /dev/null +++ b/force-app/main/default/classes/UTIL_pkBanding.cls-meta.xml @@ -0,0 +1,5 @@ + + + 52.0 + Active + diff --git a/force-app/main/default/classes/pkBandingService.cls b/force-app/main/default/classes/pkBandingService.cls new file mode 100644 index 00000000000..7becb616e15 --- /dev/null +++ b/force-app/main/default/classes/pkBandingService.cls @@ -0,0 +1,254 @@ +@RestResource(urlMapping='/pkbanding') +global with sharing class pkBandingService { + + // consts + public static final String ERR_CLASS_NAME = 'pkBandingService:'; + public static final Integer HTTP_SUCCESS = 200; + public static final Integer HTTP_FORBIDDEN = 403; + + @HttpPost + global static void pkbanding() { + + // set default response + RestResponse response = RestContext.response; + response.statusCode = HTTP_SUCCESS; + + // aux vars + Cursor request; + dispatcherResponse dResponse = new dispatcherResponse(); + SObject[] events; + + // class vars + Integer failedCount = 0; + Map failedIdsMap = new Map(); + Set failedIds = new Set(); + + // aux vars + Integer band; + Integer eventLimit; + String lastOffset; + + // rd2 evaluation service instance + RD2_OpportunityEvaluationService evalService; + + try { + + // deserialize payload + request = (Cursor) JSON.deserialize( + RestContext.request.requestBody.toString(), + Cursor.class + ); + + // cursor logic + if (request!=null && request.isCursor) { + + // init aux vars + band = request.band; + eventLimit = request.batchSize; + lastOffset = request.offset; + + // init response + dResponse.init(request); + + // init pk banding service + UTIL_pkBanding pkb = new UTIL_pkBanding(); + // set resp obj + pkb.resp = dResponse; + // get next batch + events = pkb.getNextBatch(); + // get resp obj from service + dResponse = pkb.resp; + dResponse.numberProcessed = events.size(); + + // process RDs if result == true + if (dResponse.numberProcessed>0 && dResponse.result) { + + // disable rollups + TDTM_Config_API.disableAllRollupTriggers(); + // execute RD service + evalService = new RD2_OpportunityEvaluationService().withBatchContext() + .withRds(UTIL_SObject.extractIds(events)) + .withCurrentDate(Date.Today()) + .evaluateOpportunities(); + + // capture after execution results + failedCount = evalService.getRecordsFailed(); + failedIds = evalService.getRecordsFailedId(); + failedIdsMap = evalService.getRecordsFailedMap(); + + } + + // re-try logic + } else if (request!=null && !request.isCursor && request.recordIds!=null && request.recordIds.size()>0 ) { + + // aux var + Set auxIds = new Set(); + Set rdIds = new Set(); + + // init response + dResponse.init(request); + dResponse.cursor.isCursor = false; + dResponse.result = true; + dResponse.completed = request.batchSize >= request.recordIds.size(); + + // if not completed, build set(recordIds) for next execution + if (!dResponse.completed) { + + // get array values to proccess in next batch + List listIds = new List (request.recordIds); + for (Integer i=request.batchSize; i because of bug after using removeAll() + rdIds = request.recordIds; + // add to cursor to process later + dResponse.cursor.recordIds = auxIds; + + } else { + // HAD-TO use aux SET because of bug after using removeAll() + rdIds = request.recordIds; + } + // set num processed + dResponse.numberProcessed = rdIds.size(); + + // disable rollups + TDTM_Config_API.disableAllRollupTriggers(); + // execute RD service + evalService = new RD2_OpportunityEvaluationService().withBatchContext() + .withRds(rdIds) + .withCurrentDate(Date.Today()) + .evaluateOpportunities(); + // capture after execution results + failedCount = evalService.getRecordsFailed(); + failedIds = evalService.getRecordsFailedId(); + failedIdsMap = evalService.getRecordsFailedMap(); + + } + + + } catch (Exception e) { + System.debug('EXCEPTION: ' + e.getMessage()); + // update response object with error details + dResponse.cursor.offset = lastOffset; + dResponse.numberProcessed = 0; + dResponse.numberErrors = failedCount; + dResponse.failedRecords = failedIdsMap; + dResponse.failedRecordsIds = failedIds; + dResponse.result = false; + dResponse.error = ERR_CLASS_NAME + e.getMessage(); + response.responseBody = Blob.valueOf(JSON.serialize(dResponse)); + return; + } + + // service response + dResponse.numberErrors = failedCount; + dResponse.failedRecords = failedIdsMap; + dResponse.failedRecordsIds = failedIds; + response.responseBody = Blob.valueOf(JSON.serialize(dResponse)); + } + + public void callAWS() { + + // helper class + awsRequest c = new awsRequest(); + c.batchSize = 20; + c.chunkBits = 15; + c.partitionBits = 3; + c.session = UserInfo.getSessionId(); + c.orgId = UserInfo.getOrganizationId(); + c.retryBatchSize = 200; + c.jobType = 'pkbanding'; + c.jobId = null; // generated by producer-lambda on success + c.host = System.URL.getOrgDomainUrl().toExternalForm(); + c.path = '/services/apexrest/pkbanding'; + c.url = c.host+c.path; + c.retry = true; + + String awsKey = 'AWS_KEY'; + String awsServiceUrl = 'AWS_ENDPOINT'; + + // set http request callout + HttpRequest request = new HttpRequest(); + request.setTimeout(90000); + request.setEndpoint(awsServiceUrl); + request.setMethod('POST'); + request.setHeader('Accept', 'application/json'); + request.setHeader('Content-Type', 'application/json'); + request.setHeader('X-API-KEY', awsKey); + request.setBody(JSON.serialize(c)); + + try { + // get response + Http http = new Http(); + HTTPResponse response = http.send(request); + String responseBody = response.getBody(); + System.debug('Worked! ' + responseBody ); + } catch (Exception e) { + System.debug('Exception: ' + e.getMessage()); + } + + } + + public class dispatcherResponse { + public Boolean result; + public String error; + public Integer numberProcessed; + public Integer numberErrors; + public Map failedRecords; + public Set failedRecordsIds; + public Boolean completed; // consumed all records in band + public Cursor cursor = new Cursor(); + + public void init(Cursor request) { + cursor = request; + result = true; + completed = false; + numberProcessed = 0; + numberErrors = 0; + error = ''; + failedRecords = new Map(); + failedRecordsIds = new Set(); + } + } + + public class Cursor { + public Integer band; // same as shard/subshard in TL + public Integer partitionBits; // Not yet used (currently 3 bits - 8 bands) + public Integer chunkBits; // Not yet used (currently ~14.90839 bits - 30752 recs/chunk) + public Integer batchSize; // Max number of records to pull + public Boolean sparse; // Will be true if last chunk in band was empty + public String offset; // Last Id queried (not necessarily processed) + public String url; + public String host; + public String path; + public String session; + public Boolean isCursor; + public String orgId; + public String jobType; + public String jobId; + public Integer retryBatchSize; + public Boolean retry; + public Set recordIds; + public String messageId; + public String threadId; + + } + + public class awsRequest { + public Integer batchSize; + public Integer partitionBits; + public Integer chunkBits; + public String session; + public String host; + public String path; + public String url; + public String orgId; + public String jobType; + public String jobId; + public Integer retryBatchSize; + public Boolean retry; + } + +} \ No newline at end of file diff --git a/force-app/main/default/classes/pkBandingService.cls-meta.xml b/force-app/main/default/classes/pkBandingService.cls-meta.xml new file mode 100644 index 00000000000..dd61d1f917e --- /dev/null +++ b/force-app/main/default/classes/pkBandingService.cls-meta.xml @@ -0,0 +1,5 @@ + + + 52.0 + Active + diff --git a/force-app/main/default/classes/pkBanding_CTRL.cls b/force-app/main/default/classes/pkBanding_CTRL.cls new file mode 100644 index 00000000000..82fcb8c073d --- /dev/null +++ b/force-app/main/default/classes/pkBanding_CTRL.cls @@ -0,0 +1,27 @@ +public with sharing class pkBanding_CTRL { + + public transient String baseUrl {get;set;} + + public pkBanding_CTRL() { + + baseUrl = System.URL.getOrgDomainUrl().toExternalForm(); + + } + + // action to clean NPSP and Org data + public Pagereference cleanOrg() { + + // delete all records +// delete [SELECT Id FROM npe03__Recurring_Donation__c LIMIT 1000]; +// delete [SELECT Id FROM Contact LIMIT 1000]; +// delete [SELECT Id FROM Account LIMIT 1000]; + delete [select id from opportunity LIMIT 5000]; + + // refresh page + PageReference reLoadPage = ApexPages.currentPage(); + reLoadPage.setRedirect(true); + return reLoadPage; + + } + +} \ No newline at end of file diff --git a/force-app/main/default/classes/pkBanding_CTRL.cls-meta.xml b/force-app/main/default/classes/pkBanding_CTRL.cls-meta.xml new file mode 100644 index 00000000000..dd61d1f917e --- /dev/null +++ b/force-app/main/default/classes/pkBanding_CTRL.cls-meta.xml @@ -0,0 +1,5 @@ + + + 52.0 + Active + diff --git a/force-app/main/default/pages/pkBanding.page b/force-app/main/default/pages/pkBanding.page new file mode 100644 index 00000000000..603da15d41d --- /dev/null +++ b/force-app/main/default/pages/pkBanding.page @@ -0,0 +1,712 @@ + + + + + + + + + + + + + + + +
+
+
+ + + +
+
+

Parallelization Engine Prototype | pk-banding

+

Welcome to the Parallelization Engine Prototype. We don't recommend using this page unless explicitly directed to by Salesforce.

+
+
+
+
+
+ + +
+ +
+
+
+ +
+

Configure:

+
+ +
+
+
+ + +
+ + +
+ + + +
+ +
+
+
+ +
+ +
+
+ +
+
+ +
+ +
+
+
+ +
+ +
+

Results:

+
+
+ + +
+
+
+ +
+
+
+ Clock:  : +
+
+ Transactions Processed: 
-
 +
+
+ Transactions Failed: 
-

+ +
+
+ Transactions per second: 
-
 +
+
+ Transactions per hour: 
-
 +
+
+ Transactions per day: 
-
 +
+
+
+
+
+
+
+ +
+

Logs:

+
+ +
+ +
+
+ +
+
+
+
+
+
+
+ + + + \ No newline at end of file diff --git a/force-app/main/default/pages/pkBanding.page-meta.xml b/force-app/main/default/pages/pkBanding.page-meta.xml new file mode 100644 index 00000000000..e968a52b30c --- /dev/null +++ b/force-app/main/default/pages/pkBanding.page-meta.xml @@ -0,0 +1,5 @@ + + + 52.0 + + \ No newline at end of file