From cc8deee7e8d1aa8cf6527ef4c3c3e7b2c962ab83 Mon Sep 17 00:00:00 2001 From: arybakov Date: Wed, 13 Sep 2023 17:18:50 -0600 Subject: [PATCH 1/2] GRAD2-1893 SUPP: Supplemental Distribution Run - Changes Required --- .../reader/DistributionRunPartitioner.java | 15 ++++++-------- ...istributionRunSupplementalPartitioner.java | 20 ++++++------------- .../service/DistributionService.java | 16 +++++++++------ 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitioner.java index 7bfdafa7..3c3e0b5e 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitioner.java @@ -1,7 +1,6 @@ package ca.bc.gov.educ.api.batchgraduation.reader; import ca.bc.gov.educ.api.batchgraduation.model.DistributionDataParallelDTO; -import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj; import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution; import ca.bc.gov.educ.api.batchgraduation.service.ParallelDataFetch; import org.slf4j.Logger; @@ -12,7 +11,10 @@ import org.springframework.beans.factory.annotation.Value; import reactor.core.publisher.Mono; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Monthly Distribution Partitioner @@ -29,23 +31,18 @@ public class DistributionRunPartitioner extends BasePartitioner { @Override public Map partition(int gridSize) { - ResponseObj res = restUtils.getTokenResponseObject(); - String accessToken = null; - if (res != null) { - accessToken = res.getAccess_token(); - } - // Clean up existing reports before running new one logger.debug("Delete School Reports for Monthly Distribution"); long startTime = System.currentTimeMillis(); restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken()); + restUtils.deleteSchoolReportRecord("", "DISTREP_SC", restUtils.getAccessToken()); long endTime = System.currentTimeMillis(); long diff = (endTime - startTime)/1000; logger.debug("Old School Reports deleted in {} sec", diff); startTime = System.currentTimeMillis(); logger.debug("Retrieve students for Monthly Distribution"); - Mono parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(accessToken); + Mono parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(restUtils.getAccessToken()); DistributionDataParallelDTO parallelDTO = parallelDTOMono.block(); List credentialList = new ArrayList<>(); if(parallelDTO != null) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunSupplementalPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunSupplementalPartitioner.java index 47d5a691..4938f404 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunSupplementalPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunSupplementalPartitioner.java @@ -1,6 +1,5 @@ package ca.bc.gov.educ.api.batchgraduation.reader; -import ca.bc.gov.educ.api.batchgraduation.model.DistributionDataParallelDTO; import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution; import ca.bc.gov.educ.api.batchgraduation.service.ParallelDataFetch; import org.slf4j.Logger; @@ -9,9 +8,10 @@ import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import reactor.core.publisher.Mono; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class DistributionRunSupplementalPartitioner extends BasePartitioner { @@ -28,29 +28,21 @@ public Map partition(int gridSize) { logger.debug("Delete School Reports for Supplemental Distribution"); long startTime = System.currentTimeMillis(); - String accessToken = restUtils.getAccessToken(); // Clean up existing reports before running new one - restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", accessToken); - restUtils.deleteSchoolReportRecord("", "DISTREP_SC", accessToken); + restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken()); + restUtils.deleteSchoolReportRecord("", "DISTREP_SC", restUtils.getAccessToken()); long endTime = System.currentTimeMillis(); long diff = (endTime - startTime)/1000; logger.debug("Old School Reports deleted in {} sec", diff); startTime = System.currentTimeMillis(); logger.debug("Retrieve students for Supplemental Distribution"); - Mono parallelDTOMono = parallelDataFetch.fetchDistributionRequiredDataYearly(accessToken); - DistributionDataParallelDTO parallelDTO = parallelDTOMono.block(); - List eligibleStudentSchoolDistricts = new ArrayList<>(); - if(parallelDTO != null) { - eligibleStudentSchoolDistricts.addAll(parallelDTO.transcriptList()); - eligibleStudentSchoolDistricts.addAll(parallelDTO.certificateList()); - } + List eligibleStudentSchoolDistricts = parallelDataFetch.fetchStudentCredentialsDistributionDataYearly(); endTime = System.currentTimeMillis(); diff = (endTime - startTime)/1000; logger.debug("Total {} eligible StudentCredentialDistributions found in {} sec", eligibleStudentSchoolDistricts.size(), diff); filterByStudentSearchRequest(eligibleStudentSchoolDistricts); if(!eligibleStudentSchoolDistricts.isEmpty()) { - filterOutDeceasedStudents(eligibleStudentSchoolDistricts); updateBatchJobHistory(createBatchJobHistory(), (long) eligibleStudentSchoolDistricts.size()); return getStringExecutionContextMap(gridSize, eligibleStudentSchoolDistricts, null); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java index 16634af9..48d88750 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java @@ -75,12 +75,16 @@ public void saveStudentCredentialDistribution(Long batchId, String jobType, Stud public void updateDistributionBatchJobStatus(Long batchId, int failedCount, String status) { log.debug("updateDistributionBatchJobStatus - retrieve the batch job history: batchId = {}", batchId); BatchGradAlgorithmJobHistoryEntity jobHistory = gradBatchHistoryService.getGradAlgorithmJobHistory(batchId); - jobHistory.setEndTime(LocalDateTime.now()); - jobHistory.setStatus(status); - jobHistory.setActualStudentsProcessed(jobHistory.getExpectedStudentsProcessed() - failedCount); - log.debug("updateDistributionBatchJobStatus - save the batch job history: batchId = {}, status = {}. actual processed count = {}", batchId, status, jobHistory.getActualStudentsProcessed()); - gradBatchHistoryService.saveGradAlgorithmJobHistory(jobHistory); - log.debug("updateDistributionBatchJobStatus - save the batch job history is completed!"); + if(jobHistory != null) { + jobHistory.setEndTime(LocalDateTime.now()); + jobHistory.setStatus(status); + jobHistory.setActualStudentsProcessed(jobHistory.getExpectedStudentsProcessed() - failedCount); + log.debug("updateDistributionBatchJobStatus - save the batch job history: batchId = {}, status = {}. actual processed count = {}", batchId, status, jobHistory.getActualStudentsProcessed()); + gradBatchHistoryService.saveGradAlgorithmJobHistory(jobHistory); + log.debug("updateDistributionBatchJobStatus - save the batch job history is completed!"); + } else { + log.debug("updateDistributionBatchJobStatus - batch job history NOT FOUND: batchId = {}", batchId); + } } } From d854b06d807b5f82d2c286044a06c4839ea5c2eb Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 14 Sep 2023 11:09:26 -0600 Subject: [PATCH 2/2] GRAD2-1893 SUPP: Supplemental Distribution Run - Changes Required --- .../service/DistributionService.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java index 48d88750..16634af9 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/DistributionService.java @@ -75,16 +75,12 @@ public void saveStudentCredentialDistribution(Long batchId, String jobType, Stud public void updateDistributionBatchJobStatus(Long batchId, int failedCount, String status) { log.debug("updateDistributionBatchJobStatus - retrieve the batch job history: batchId = {}", batchId); BatchGradAlgorithmJobHistoryEntity jobHistory = gradBatchHistoryService.getGradAlgorithmJobHistory(batchId); - if(jobHistory != null) { - jobHistory.setEndTime(LocalDateTime.now()); - jobHistory.setStatus(status); - jobHistory.setActualStudentsProcessed(jobHistory.getExpectedStudentsProcessed() - failedCount); - log.debug("updateDistributionBatchJobStatus - save the batch job history: batchId = {}, status = {}. actual processed count = {}", batchId, status, jobHistory.getActualStudentsProcessed()); - gradBatchHistoryService.saveGradAlgorithmJobHistory(jobHistory); - log.debug("updateDistributionBatchJobStatus - save the batch job history is completed!"); - } else { - log.debug("updateDistributionBatchJobStatus - batch job history NOT FOUND: batchId = {}", batchId); - } + jobHistory.setEndTime(LocalDateTime.now()); + jobHistory.setStatus(status); + jobHistory.setActualStudentsProcessed(jobHistory.getExpectedStudentsProcessed() - failedCount); + log.debug("updateDistributionBatchJobStatus - save the batch job history: batchId = {}, status = {}. actual processed count = {}", batchId, status, jobHistory.getActualStudentsProcessed()); + gradBatchHistoryService.saveGradAlgorithmJobHistory(jobHistory); + log.debug("updateDistributionBatchJobStatus - save the batch job history is completed!"); } }