From 77430956ad6d4b3164213ab9b4c4d11614658438 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:01:04 -0700 Subject: [PATCH 1/4] chore(deps): update bcgov/quickstart-openshift-helpers action to v0.8.0 (#1668) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/pr-close.yml | 2 +- .github/workflows/pr-validate.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-close.yml b/.github/workflows/pr-close.yml index ec50494ba..a70060095 100644 --- a/.github/workflows/pr-close.yml +++ b/.github/workflows/pr-close.yml @@ -12,7 +12,7 @@ concurrency: jobs: cleanup: name: Cleanup and Images - uses: bcgov/quickstart-openshift-helpers/.github/workflows/.pr-close.yml@v0.7.1 + uses: bcgov/quickstart-openshift-helpers/.github/workflows/.pr-close.yml@v0.8.0 secrets: oc_namespace: ${{ vars.OC_NAMESPACE }} oc_token: ${{ secrets.OC_TOKEN }} diff --git a/.github/workflows/pr-validate.yml b/.github/workflows/pr-validate.yml index 87f5a28fa..c4507e02b 100644 --- a/.github/workflows/pr-validate.yml +++ b/.github/workflows/pr-validate.yml @@ -22,7 +22,7 @@ jobs: validate: name: Validate PR needs: [init] - uses: bcgov/quickstart-openshift-helpers/.github/workflows/.pr-validate.yml@v0.7.1 + uses: bcgov/quickstart-openshift-helpers/.github/workflows/.pr-validate.yml@v0.8.0 with: markdown_links: | - [Frontend](https://${{ github.event.repository.name }}-${{ needs.init.outputs.mod-tag }}-frontend.apps.silver.devops.gov.bc.ca/) From 064913cf50a1897754344d9f412eabd2b3eb02e5 Mon Sep 17 00:00:00 2001 From: Ricardo Campos Date: Fri, 11 Oct 2024 16:49:14 -0300 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20handle=20possible=20null=20for=20the?= =?UTF-8?q?=20contaminantPollenBv=20recalculate=20but=E2=80=A6=20(#1669)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ca/bc/gov/backendstartapi/dto/PtValsCalReqDto.java | 2 +- .../gov/backendstartapi/service/ParentTreeService.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/ca/bc/gov/backendstartapi/dto/PtValsCalReqDto.java b/backend/src/main/java/ca/bc/gov/backendstartapi/dto/PtValsCalReqDto.java index 94cff9744..6d22d0a4a 100644 --- a/backend/src/main/java/ca/bc/gov/backendstartapi/dto/PtValsCalReqDto.java +++ b/backend/src/main/java/ca/bc/gov/backendstartapi/dto/PtValsCalReqDto.java @@ -13,4 +13,4 @@ public record PtValsCalReqDto( @NotNull List orchardPtVals, @NotNull List smpMixIdAndProps, @NotNull Integer smpParentsOutside, - @NotNull BigDecimal contaminantPollenBv) {} + BigDecimal contaminantPollenBv) {} diff --git a/backend/src/main/java/ca/bc/gov/backendstartapi/service/ParentTreeService.java b/backend/src/main/java/ca/bc/gov/backendstartapi/service/ParentTreeService.java index 1a05562c3..400d110e8 100644 --- a/backend/src/main/java/ca/bc/gov/backendstartapi/service/ParentTreeService.java +++ b/backend/src/main/java/ca/bc/gov/backendstartapi/service/ParentTreeService.java @@ -203,7 +203,12 @@ public PtCalculationResDto calculatePtVals(PtValsCalReqDto ptVals) { Integer totalNonOrchardPollen = 0; Integer numNonOrchardPollen = 0; - SparLog.debug("ptVals.contaminantPollenBv(): {}", ptVals.contaminantPollenBv()); + double contaminantPollenBvDouble = 0; + if (ValueUtil.hasValue(ptVals.contaminantPollenBv())) { + contaminantPollenBvDouble = ptVals.contaminantPollenBv().doubleValue(); + } + + SparLog.debug("contaminantPollenBvDouble: {}", contaminantPollenBvDouble); // --Third pass to calc values that depend on totals derived above and the remainder for (OrchardParentTreeValsDto parentTreeRow : ptVals.orchardPtVals()) { @@ -257,7 +262,7 @@ public PtCalculationResDto calculatePtVals(PtValsCalReqDto ptVals) { // --col:AA double vmContamContrib = - (auxValueAa * ptVals.contaminantPollenBv().doubleValue()) * femaleCropPop.doubleValue(); + (auxValueAa * contaminantPollenBvDouble) * femaleCropPop.doubleValue(); // --col:AB (depends on SUM(X)=v_sum_m_gw_contrib_orch_poll) double auxValueAb = From ecb361b282606aec01ea592e2386762ea7a24c44 Mon Sep 17 00:00:00 2001 From: Ricardo Campos Date: Fri, 11 Oct 2024 19:20:37 -0300 Subject: [PATCH 3/4] feat: add backup routine for SPAR postgres DB (#1556) Co-authored-by: Derek Roberts --- .github/workflows/.deploy.yml | 13 +- common/openshift.backup.yml | 239 ++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 common/openshift.backup.yml diff --git a/.github/workflows/.deploy.yml b/.github/workflows/.deploy.yml index b3ee203e6..49c0dc175 100644 --- a/.github/workflows/.deploy.yml +++ b/.github/workflows/.deploy.yml @@ -114,24 +114,26 @@ jobs: timeout-minutes: 10 strategy: matrix: - name: [backend, frontend, oracle-api] + name: [backend, backup, frontend, oracle-api] include: - name: backend file: backend/openshift.deploy.yml - overwrite: true parameters: -p AWS_COGNITO_ISSUER_URI=https://cognito-idp.ca-central-1.amazonaws.com/${{ vars.VITE_USER_POOLS_ID }} verification_path: "health" + - name: backup + file: common/openshift.backup.yml + parameters: + -p PG_DB_IMAGE=postgis/postgis:15-master + post_rollout: oc create job --from=cronjob/${{ github.event.repository.name }}-${{ inputs.target }}-database-backup ${{ github.event.repository.name }}-${{ inputs.target }}-database-backup-$(date +%Y%m%d%H%M%S) - name: frontend file: frontend/openshift.deploy.yml - overwrite: true parameters: -p FAM_MODDED_ZONE=${{ needs.init.outputs.fam-modded-zone }} -p VITE_SPAR_BUILD_VERSION=snapshot-${{ inputs.target || github.event.number }} -p VITE_USER_POOLS_ID=${{ vars.VITE_USER_POOLS_ID }} - name: oracle-api file: oracle-api/openshift.deploy.yml - overwrite: true parameters: -p AWS_COGNITO_ISSUER_URI=https://cognito-idp.ca-central-1.amazonaws.com/${{ vars.VITE_USER_POOLS_ID }} ${{ github.event_name == 'pull_request' && '-p CPU_LIMIT=100m' || '' }} @@ -147,13 +149,14 @@ jobs: oc_namespace: ${{ vars.OC_NAMESPACE }} oc_server: ${{ vars.OC_SERVER }} oc_token: ${{ secrets.OC_TOKEN }} - overwrite: ${{ matrix.overwrite }} + overwrite: true parameters: -p TAG=${{ inputs.tag }} -p ZONE=${{ inputs.target }} ${{ github.event_name == 'pull_request' && '-p MIN_REPLICAS=1' || '' }} ${{ github.event_name == 'pull_request' && '-p MAX_REPLICAS=1' || '' }} ${{ matrix.parameters }} + post_rollout: ${{ matrix.post_rollout || '' }} verification_path: ${{ matrix.verification_path }} verification_retry_attempts: 5 verification_retry_seconds: 20 diff --git a/common/openshift.backup.yml b/common/openshift.backup.yml new file mode 100644 index 000000000..c5ae2b3d5 --- /dev/null +++ b/common/openshift.backup.yml @@ -0,0 +1,239 @@ +apiVersion: template.openshift.io/v1 +kind: Template +labels: + app: ${NAME}-${ZONE} + app.kubernetes.io/part-of: ${NAME}-${ZONE} +parameters: + - name: NAME + description: Product name + value: nr-spar + - name: COMPONENT + description: Component name + value: database-backup + - name: ZONE + description: Deployment zone, e.g. pr-### or prod + required: true + - name: RESTORE_DIR + description: Directory to be used for restoring the backup + value: /tmp/restore + - name: REGISTRY + description: Container registry to import from (internal is image-registry.openshift-image-registry.svc:5000) + value: ghcr.io + - name: BACKUP_DIR + description: "The name of the root backup directory" + required: true + value: /tmp/backup + - name: NUM_BACKUPS + description: The number of backup files to be retained + required: false + value: "5" + - name: "JOB_SERVICE_ACCOUNT" + description: "Name of the Service Account To Exeucte the Job As." + value: "default" + required: true + - name: "SUCCESS_JOBS_HISTORY_LIMIT" + description: "The number of successful jobs that will be retained" + value: "5" + required: true + - name: "FAILED_JOBS_HISTORY_LIMIT" + description: "The number of failed jobs that will be retained" + value: "2" + required: true + - name: "JOB_BACKOFF_LIMIT" + description: "The number of attempts to try for a successful job outcome" + value: "0" + - name: PVC_SIZE + description: Volume space available for data, e.g. 512Mi, 2Gi. + value: 256Mi + - name: PG_DB_IMAGE + description: PostgreSQL Image (namespace/name:tag) to be used for backup + required: true + - name: TAG + description: Dummy param, for convenience + - name: MIN_REPLICAS + description: Dummy param, for convenience + - name: MAX_REPLICAS + description: Dummy param, for convenience +objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: ${NAME}-${ZONE}-${COMPONENT} + labels: + app: ${NAME}-${ZONE} + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "${PVC_SIZE}" + storageClassName: netapp-file-standard + - kind: CronJob + apiVersion: "batch/v1" + metadata: + name: ${NAME}-${ZONE}-${COMPONENT} + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + schedule: "0 0 * * *" + concurrencyPolicy: "Replace" + successfulJobsHistoryLimit: ${{SUCCESS_JOBS_HISTORY_LIMIT}} + failedJobsHistoryLimit: ${{FAILED_JOBS_HISTORY_LIMIT}} + jobTemplate: + metadata: + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + backoffLimit: ${{JOB_BACKOFF_LIMIT}} + template: + metadata: + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + containers: + - name: ${NAME}-${ZONE}-${COMPONENT} + image: ${REGISTRY}/${PG_DB_IMAGE} + command: ["/bin/sh", "-c"] + args: + - | + pg_dump \ + -U ${POSTGRESQL_USER} \ + -h ${NAME}-${ZONE}-database \ + -d ${POSTGRESQL_DATABASE} \ + --data-only \ + --schema=nr-spar \ + --inserts \ + --no-comments \ + --on-conflict-do-nothing \ + --no-sync \ + --exclude-table=nr-spar.cone_collection_method_list \ + --exclude-table=nr-spar.gametic_methodology_list \ + --exclude-table=nr-spar.genetic_class_list \ + --exclude-table=nr-spar.genetic_worth_list \ + --exclude-table=nr-spar.method_of_payment_list \ + --exclude-table=nr-spar.seedlot_source_list \ + --exclude-table=nr-spar.seedlot_status_list \ + --exclude-table=nr-spar.etl_execution_log_hist \ + --exclude-table=nr-spar.etl_execution_map \ + --exclude-table=nr-spar.etl_execution_schedule \ + --file=${BACKUP_DIR}/backup_$(date +%Y-%m-%d).sql \ + && + find "${BACKUP_DIR}" -type f -mtime +$NUM_BACKUPS -exec rm -f {} \; && + cp -r ${BACKUP_DIR}/backup_$(date +%Y-%m-%d).sql ${RESTORE_DIR}/W0__restore.sql + volumeMounts: + - mountPath: "${BACKUP_DIR}" + name: ${NAME}-${ZONE}-${COMPONENT} + - mountPath: "${RESTORE_DIR}" + name: ${NAME}-${ZONE}-${COMPONENT} + env: + - name: RESTORE_DIR + value: "${RESTORE_DIR}" + - name: BACKUP_DIR + value: "${BACKUP_DIR}" + - name: NUM_BACKUPS + value: "${NUM_BACKUPS}" + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-name + - name: POSTGRESQL_USER + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-user + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-password + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-password + volumes: + - name: ${NAME}-${ZONE}-${COMPONENT} + persistentVolumeClaim: + claimName: ${NAME}-${ZONE}-${COMPONENT} + restartPolicy: "Never" + terminationGracePeriodSeconds: 30 + activeDeadlineSeconds: 1600 + dnsPolicy: "ClusterFirst" + serviceAccountName: "${JOB_SERVICE_ACCOUNT}" + serviceAccount: "${JOB_SERVICE_ACCOUNT}" + - kind: CronJob + apiVersion: "batch/v1" + metadata: + name: ${NAME}-${ZONE}-${COMPONENT}-restore + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + schedule: "0 0 31 2 *" + concurrencyPolicy: "Replace" + successfulJobsHistoryLimit: ${{SUCCESS_JOBS_HISTORY_LIMIT}} + failedJobsHistoryLimit: ${{FAILED_JOBS_HISTORY_LIMIT}} + jobTemplate: + metadata: + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + backoffLimit: ${{JOB_BACKOFF_LIMIT}} + template: + metadata: + labels: + app: ${NAME}-${ZONE} + cronjob: ${NAME}-${ZONE} + spec: + containers: + - name: ${NAME}-${ZONE}-${COMPONENT}-restore + image: ${REGISTRY}/${PG_DB_IMAGE} + command: ["/bin/sh", "-c"] + args: + - | + find ${RESTORE_DIR} -type f -name "*.sql" -print0 | sort -zV | + while IFS= read -r -d '' sql_file; do + echo "Running SQL file: $sql_file" + psql -h ${TARGET_HOST} -U ${POSTGRESQL_USER} -d ${POSTGRESQL_DATABASE} -f $sql_file + done + volumeMounts: + - mountPath: "${RESTORE_DIR}" + name: ${NAME}-${ZONE}-${COMPONENT} + env: + - name: RESTORE_DIR + value: "${RESTORE_DIR}" + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-name + - name: POSTGRESQL_USER + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-user + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-password + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-password + volumes: + - name: ${NAME}-${ZONE}-${COMPONENT} + persistentVolumeClaim: + claimName: ${NAME}-${ZONE}-${COMPONENT} + restartPolicy: "Never" + terminationGracePeriodSeconds: 30 + activeDeadlineSeconds: 1600 + dnsPolicy: "ClusterFirst" + serviceAccountName: "${JOB_SERVICE_ACCOUNT}" + serviceAccount: "${JOB_SERVICE_ACCOUNT}" From 766f73f2d557ed8d6ceb3becc0bd918aec31e9ff Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:50:33 -0700 Subject: [PATCH 4/4] chore(deps): update aquasecurity/trivy-action action to v0.25.0 (#1672) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 8b3d324f5..7e8cf5484 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -60,7 +60,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@0.24.0 + uses: aquasecurity/trivy-action@0.25.0 with: format: "sarif" output: "trivy-results.sarif"