From 3998c8fbf20f3e0cd8ff30a1641665b52831fcb2 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 09:53:12 -0700 Subject: [PATCH 01/18] Dial down database resources --- database/openshift.deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/database/openshift.deploy.yml b/database/openshift.deploy.yml index 9daa3e99a..50221107c 100644 --- a/database/openshift.deploy.yml +++ b/database/openshift.deploy.yml @@ -25,13 +25,13 @@ parameters: description: Where to mount the PVC, subpath (e.g. data/) value: /var/lib/postgresql - name: CPU_REQUEST - value: 50m + value: 25m - name: CPU_LIMIT - value: 115m + value: 75m - name: MEMORY_REQUEST - value: 150Mi + value: 100Mi - name: MEMORY_LIMIT - value: 250Mi + value: 200Mi - name: DB_PVC_SIZE description: Volume space available for data, e.g. 512Mi, 2Gi. displayName: Database Volume Capacity From 9ff9bb9f6cfe549f3550217eb9c3390e64214a45 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 10:16:37 -0700 Subject: [PATCH 02/18] Revise backend resources --- backend/openshift.deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index bfcb69af5..da585b0c2 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,13 +35,13 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 100m + value: 15m - name: CPU_LIMIT - value: 300m + value: 30m - name: MEMORY_REQUEST - value: 100Mi + value: 200Mi - name: MEMORY_LIMIT - value: 500Mi + value: 300Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From b656648b23e5726061d0c6a5ad387e6792f2fa28 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 10:20:09 -0700 Subject: [PATCH 03/18] Revise oracle-api resources --- oracle-api/openshift.deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oracle-api/openshift.deploy.yml b/oracle-api/openshift.deploy.yml index a6033b7e6..c8fc8028d 100644 --- a/oracle-api/openshift.deploy.yml +++ b/oracle-api/openshift.deploy.yml @@ -24,13 +24,13 @@ parameters: - name: DOMAIN value: apps.silver.devops.gov.bc.ca - name: CPU_REQUEST - value: 100m + value: 15m - name: CPU_LIMIT - value: 500m + value: 30m - name: MEMORY_REQUEST - value: 300Mi + value: 150Mi - name: MEMORY_LIMIT - value: 500Mi + value: 225Mi - name: ALLOWED_ORIGINS description: Sets all the allowed request origins value: "http://localhost:300*,https://*.apps.silver.devops.gov.bc.ca,https://*.nrs.gov.bc.ca" From ac6d26b343f582b51f676fd203d0373f448d0e42 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 10:33:00 -0700 Subject: [PATCH 04/18] Dial down frontend resources --- frontend/openshift.deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/openshift.deploy.yml b/frontend/openshift.deploy.yml index c65a94039..b663ba42d 100644 --- a/frontend/openshift.deploy.yml +++ b/frontend/openshift.deploy.yml @@ -23,7 +23,7 @@ parameters: value: ghcr.io - name: ORG description: Organization name - value: bcgov + value: bcgov - name: DOMAIN value: apps.silver.devops.gov.bc.ca - name: CPU_REQUEST @@ -31,9 +31,9 @@ parameters: - name: CPU_LIMIT value: 30m - name: MEMORY_REQUEST - value: 30Mi + value: 20Mi - name: MEMORY_LIMIT - value: 50Mi + value: 40Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From c4ee4e5a7f074f2cf95713ef2bea329a018ee7a6 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 10:41:40 -0700 Subject: [PATCH 05/18] Add autoscaling for memory --- backend/openshift.deploy.yml | 6 ++++++ frontend/openshift.deploy.yml | 6 ++++++ oracle-api/openshift.deploy.yml | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index da585b0c2..df56bedc7 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -204,3 +204,9 @@ objects: target: type: Utilization averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 diff --git a/frontend/openshift.deploy.yml b/frontend/openshift.deploy.yml index b663ba42d..1c7035d19 100644 --- a/frontend/openshift.deploy.yml +++ b/frontend/openshift.deploy.yml @@ -166,6 +166,12 @@ objects: target: type: Utilization averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 - apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/oracle-api/openshift.deploy.yml b/oracle-api/openshift.deploy.yml index c8fc8028d..f423c678a 100644 --- a/oracle-api/openshift.deploy.yml +++ b/oracle-api/openshift.deploy.yml @@ -236,3 +236,9 @@ objects: target: type: Utilization averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 From 9aa8f947d5c606742107032cf0c2540dfe852a6f Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 11:08:25 -0700 Subject: [PATCH 06/18] Dial up Java CPU limits --- backend/openshift.deploy.yml | 2 +- oracle-api/openshift.deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index df56bedc7..8fdd87d16 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -37,7 +37,7 @@ parameters: - name: CPU_REQUEST value: 15m - name: CPU_LIMIT - value: 30m + value: 50m - name: MEMORY_REQUEST value: 200Mi - name: MEMORY_LIMIT diff --git a/oracle-api/openshift.deploy.yml b/oracle-api/openshift.deploy.yml index f423c678a..fabf4de54 100644 --- a/oracle-api/openshift.deploy.yml +++ b/oracle-api/openshift.deploy.yml @@ -26,7 +26,7 @@ parameters: - name: CPU_REQUEST value: 15m - name: CPU_LIMIT - value: 30m + value: 50m - name: MEMORY_REQUEST value: 150Mi - name: MEMORY_LIMIT From 0df4b49c55dc1599b87a878f9fbcceb657639cde Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 16:19:44 -0700 Subject: [PATCH 07/18] Keep revising backend numbers --- backend/openshift.deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 8fdd87d16..5c026eaad 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,13 +35,13 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 15m + value: 100m - name: CPU_LIMIT - value: 50m + value: 300m - name: MEMORY_REQUEST value: 200Mi - name: MEMORY_LIMIT - value: 300Mi + value: 500Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From 31f119a11bffbb30c3b53220cd0b8a2cdc6dbb08 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 16:21:20 -0700 Subject: [PATCH 08/18] Pare down workflow while testing --- .github/workflows/pr-open.yml | 83 +++++++++++++++++------------------ 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index e72b6ec6e..bb066d7d6 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -9,54 +9,51 @@ concurrency: cancel-in-progress: true jobs: - builds: - name: Builds - runs-on: ubuntu-22.04 - outputs: - triggered: ${{ steps.build.outputs.triggered }} - permissions: - packages: write - strategy: - matrix: - package: [database, common, backend, frontend, oracle-api, sync] - steps: - - uses: bcgov-nr/action-builder-ghcr@v2.2.0 - id: build - with: - build_args: | - "BUILDKIT_INLINE_CACHE=1" - --build-arg BUILD_NUMBER=${{ github.event.number }} - package: ${{ matrix.package }} - tag: ${{ github.event.number }} - tag_fallback: latest - triggers: ('${{ matrix.package }}/') + # builds: + # name: Builds + # runs-on: ubuntu-22.04 + # outputs: + # triggered: ${{ steps.build.outputs.triggered }} + # permissions: + # packages: write + # strategy: + # matrix: + # package: [database, common, backend, frontend, oracle-api, sync] + # steps: + # - uses: bcgov-nr/action-builder-ghcr@v2.2.0 + # id: build + # with: + # package: ${{ matrix.package }} + # tag: ${{ github.event.number }} + # tag_fallback: latest + # triggers: ('${{ matrix.package }}/') deploys: name: Deploys - needs: [builds] + # needs: [builds] secrets: inherit uses: ./.github/workflows/.deploy.yml - tests: - name: Tests - if: needs.deploys.outputs.run_tests == 'true' - needs: [deploys] - secrets: inherit - uses: ./.github/workflows/.tests.yml + # tests: + # name: Tests + # if: needs.deploys.outputs.run_tests == 'true' + # needs: [deploys] + # secrets: inherit + # uses: ./.github/workflows/.tests.yml - results: - name: PR Results - if: always() - # Include all needs that could have failures! - needs: [builds, deploys, tests] - runs-on: ubuntu-22.04 - steps: - - run: | - # View results - echo "needs.*.result: ${{ toJson(needs.*.result) }}" + # results: + # name: PR Results + # if: always() + # # Include all needs that could have failures! + # needs: [builds, deploys, tests] + # runs-on: ubuntu-22.04 + # steps: + # - run: | + # # View results + # echo "needs.*.result: ${{ toJson(needs.*.result) }}" - - if: contains(needs.*.result, 'failure') - run: | - # Job failure found - echo "At least one job has failed" - exit 1 + # - if: contains(needs.*.result, 'failure') + # run: | + # # Job failure found + # echo "At least one job has failed" + # exit 1 From 3dd9ea9bb3f89a786abcdaa1a425724c558691e5 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 16:29:33 -0700 Subject: [PATCH 09/18] Dial down backend memory --- backend/openshift.deploy.yml | 4 ++-- frontend/openshift.deploy.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 5c026eaad..93b918a1c 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -39,9 +39,9 @@ parameters: - name: CPU_LIMIT value: 300m - name: MEMORY_REQUEST - value: 200Mi + value: 100Mi - name: MEMORY_LIMIT - value: 500Mi + value: 300Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" diff --git a/frontend/openshift.deploy.yml b/frontend/openshift.deploy.yml index 1c7035d19..ea4094dec 100644 --- a/frontend/openshift.deploy.yml +++ b/frontend/openshift.deploy.yml @@ -23,7 +23,7 @@ parameters: value: ghcr.io - name: ORG description: Organization name - value: bcgov + value: bcgov - name: DOMAIN value: apps.silver.devops.gov.bc.ca - name: CPU_REQUEST From 4d60d26a53c1a0f712b6e65d1f52a7a1d5b05ca4 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 2 Aug 2024 16:56:17 -0700 Subject: [PATCH 10/18] Keep at it --- backend/openshift.deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 93b918a1c..1284ec003 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -41,7 +41,7 @@ parameters: - name: MEMORY_REQUEST value: 100Mi - name: MEMORY_LIMIT - value: 300Mi + value: 500Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From 580a5575283481c365a86b1e89fe4079d3d7633a Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 6 Aug 2024 09:36:11 -0700 Subject: [PATCH 11/18] Dialed backend back down --- backend/openshift.deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 1284ec003..766fc1360 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,13 +35,13 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 100m + value: 150m - name: CPU_LIMIT - value: 300m + value: 50m - name: MEMORY_REQUEST value: 100Mi - name: MEMORY_LIMIT - value: 500Mi + value: 300Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From e1b726510edf2fa85b341ae06e9f03423d342d51 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 6 Aug 2024 09:44:52 -0700 Subject: [PATCH 12/18] Resources --- backend/openshift.deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 766fc1360..c4db16d48 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,9 +35,9 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 150m - - name: CPU_LIMIT value: 50m + - name: CPU_LIMIT + value: 100m - name: MEMORY_REQUEST value: 100Mi - name: MEMORY_LIMIT From 948e73345fb0533865f00e9545b101600392ded4 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 6 Aug 2024 12:50:18 -0700 Subject: [PATCH 13/18] Set backend RAM limit to 3x request --- backend/openshift.deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index c4db16d48..09d7a190e 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -39,9 +39,9 @@ parameters: - name: CPU_LIMIT value: 100m - name: MEMORY_REQUEST - value: 100Mi + value: 150Mi - name: MEMORY_LIMIT - value: 300Mi + value: 450Mi - name: MIN_REPLICAS description: The minimum amount of replicas for the horizontal pod autoscaler. value: "3" From ffe727e9c935dc34f9c474022d150dcc5b7c62bd Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 6 Aug 2024 13:16:26 -0700 Subject: [PATCH 14/18] Set backend CPU limit to 3x request --- backend/openshift.deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 09d7a190e..9b628bd26 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,9 +35,9 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 50m + value: 10m - name: CPU_LIMIT - value: 100m + value: 30m - name: MEMORY_REQUEST value: 150Mi - name: MEMORY_LIMIT From f5c1b491351f337d2c61ab393ef8d8840448ac8f Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 13 Aug 2024 12:55:41 -0700 Subject: [PATCH 15/18] Keep messing with backend --- oracle-api/openshift.deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle-api/openshift.deploy.yml b/oracle-api/openshift.deploy.yml index fabf4de54..f8c598b1b 100644 --- a/oracle-api/openshift.deploy.yml +++ b/oracle-api/openshift.deploy.yml @@ -26,7 +26,7 @@ parameters: - name: CPU_REQUEST value: 15m - name: CPU_LIMIT - value: 50m + value: 100m - name: MEMORY_REQUEST value: 150Mi - name: MEMORY_LIMIT From 9e9c0ea5246c4d289436a8a9e89ded0ac7acd065 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 13 Aug 2024 13:06:26 -0700 Subject: [PATCH 16/18] Keep messing with backend --- backend/openshift.deploy.yml | 4 ++-- oracle-api/openshift.deploy.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/openshift.deploy.yml b/backend/openshift.deploy.yml index 9b628bd26..bf6154967 100644 --- a/backend/openshift.deploy.yml +++ b/backend/openshift.deploy.yml @@ -35,9 +35,9 @@ parameters: - name: FORESTCLIENTAPI_ADDRESS value: "https://nr-forest-client-api-prod.api.gov.bc.ca/api" - name: CPU_REQUEST - value: 10m + value: 15m - name: CPU_LIMIT - value: 30m + value: 60m - name: MEMORY_REQUEST value: 150Mi - name: MEMORY_LIMIT diff --git a/oracle-api/openshift.deploy.yml b/oracle-api/openshift.deploy.yml index f8c598b1b..fabf4de54 100644 --- a/oracle-api/openshift.deploy.yml +++ b/oracle-api/openshift.deploy.yml @@ -26,7 +26,7 @@ parameters: - name: CPU_REQUEST value: 15m - name: CPU_LIMIT - value: 100m + value: 50m - name: MEMORY_REQUEST value: 150Mi - name: MEMORY_LIMIT From c7792a01346bc224ee02604c03821f0424e9e51c Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 13 Aug 2024 13:27:12 -0700 Subject: [PATCH 17/18] Restore workflows --- .github/workflows/pr-open.yml | 80 +++++++++++++++++------------------ 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index bb066d7d6..d6e8bde2b 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -9,51 +9,51 @@ concurrency: cancel-in-progress: true jobs: - # builds: - # name: Builds - # runs-on: ubuntu-22.04 - # outputs: - # triggered: ${{ steps.build.outputs.triggered }} - # permissions: - # packages: write - # strategy: - # matrix: - # package: [database, common, backend, frontend, oracle-api, sync] - # steps: - # - uses: bcgov-nr/action-builder-ghcr@v2.2.0 - # id: build - # with: - # package: ${{ matrix.package }} - # tag: ${{ github.event.number }} - # tag_fallback: latest - # triggers: ('${{ matrix.package }}/') + builds: + name: Builds + runs-on: ubuntu-22.04 + outputs: + triggered: ${{ steps.build.outputs.triggered }} + permissions: + packages: write + strategy: + matrix: + package: [database, common, backend, frontend, oracle-api, sync] + steps: + - uses: bcgov-nr/action-builder-ghcr@v2.2.0 + id: build + with: + package: ${{ matrix.package }} + tag: ${{ github.event.number }} + tag_fallback: latest + triggers: ('${{ matrix.package }}/') deploys: name: Deploys - # needs: [builds] + needs: [builds] secrets: inherit uses: ./.github/workflows/.deploy.yml - # tests: - # name: Tests - # if: needs.deploys.outputs.run_tests == 'true' - # needs: [deploys] - # secrets: inherit - # uses: ./.github/workflows/.tests.yml + tests: + name: Tests + if: needs.deploys.outputs.run_tests == 'true' + needs: [deploys] + secrets: inherit + uses: ./.github/workflows/.tests.yml - # results: - # name: PR Results - # if: always() - # # Include all needs that could have failures! - # needs: [builds, deploys, tests] - # runs-on: ubuntu-22.04 - # steps: - # - run: | - # # View results - # echo "needs.*.result: ${{ toJson(needs.*.result) }}" + results: + name: PR Results + if: always() + # Include all needs that could have failures! + needs: [builds, deploys, tests] + runs-on: ubuntu-22.04 + steps: + - run: | + # View results + echo "needs.*.result: ${{ toJson(needs.*.result) }}" - # - if: contains(needs.*.result, 'failure') - # run: | - # # Job failure found - # echo "At least one job has failed" - # exit 1 + - if: contains(needs.*.result, 'failure') + run: | + # Job failure found + echo "At least one job has failed" + exit 1 From c90e9b8456b0050c3601fc13cbaa134078f1dfba Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 13 Aug 2024 13:28:04 -0700 Subject: [PATCH 18/18] Restore build_args --- .github/workflows/pr-open.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index d6e8bde2b..e72b6ec6e 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -23,6 +23,9 @@ jobs: - uses: bcgov-nr/action-builder-ghcr@v2.2.0 id: build with: + build_args: | + "BUILDKIT_INLINE_CACHE=1" + --build-arg BUILD_NUMBER=${{ github.event.number }} package: ${{ matrix.package }} tag: ${{ github.event.number }} tag_fallback: latest