feat: Release Spirit Bear

.github/workflows/.dbdeployer.yml

@@ -5,12 +5,12 @@ on:
     inputs: ### Required
       directory:
         description: Crunchy Chart directory
-        default: 'charts/crunchy'
+        default: "charts/crunchy"
         required: false
         type: string
       oc_server:
         default: https://api.silver.devops.gov.bc.ca:6443
-        description: 'OpenShift server'
+        description: "OpenShift server"
         required: false
         type: string
       environment:
@@ -21,29 +21,29 @@ on:
         description: Cluster environment name, should be dev,test,prod
         required: false
         type: string
-        default: 'dev'
+        default: "dev"
       s3_enabled:
         description: Enable S3 backups
         required: false
         default: true
         type: boolean
       values:
-        description: 'Values file'
-        default: 'values.yaml'
+        description: "Values file"
+        default: "values.yaml"
         required: false
         type: string
       app_values:
-        description: 'App specific values file which is present inside charts/app'
-        default: 'values.yaml'
+        description: "App specific values file which is present inside charts/app"
+        default: "values.yaml"
         required: false
         type: string
       enabled:
-        description: 'Enable the deployment of the crunchy database, easy switch to turn it on/off'
+        description: "Enable the deployment of the crunchy database, easy switch to turn it on/off"
         default: true
         required: false
         type: boolean
       timeout-minutes:
-        description: 'Timeout minutes'
+        description: "Timeout minutes"
         default: 20
         required: false
         type: number
@@ -52,8 +52,8 @@ on:
         required: false
         type: string
       params:
-        description: 'Extra parameters to pass to helm upgrade'
-        default: ''
+        description: "Extra parameters to pass to helm upgrade"
+        default: ""
         required: false
         type: string
     secrets:
@@ -88,7 +88,7 @@ jobs:
         uses: redhat-actions/openshift-tools-installer@v1
         with:
           oc: "4.14.37"
-      - uses: bcgov-nr/[email protected]
+      - uses: bcgov/[email protected]
         id: triggers
         with:
           triggers: ${{ inputs.triggers }}
@@ -146,47 +146,44 @@ jobs:
         shell: bash
         if: github.event_name == 'pull_request'
         run: |
-            echo 'Adding PR specific user to Crunchy DB'
-            NEW_USER='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
-            CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} -o json | jq '.spec.users')
-            echo "${CURRENT_USERS}"
+          echo 'Adding PR specific user to Crunchy DB'
+          NEW_USER='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
+          CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} -o json | jq '.spec.users')
+          echo "${CURRENT_USERS}"
 
-            # check if current_users already contains the new_user
-            if echo "${CURRENT_USERS}" | jq -e ".[] | select(.name == \"app-${{ github.event.number }}\")" > /dev/null; then
-              echo "User already exists"
-              exit 0
-            fi
-
-            UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson NEW_USER "$NEW_USER" '. + [$NEW_USER]')
-            echo "$UPDATED_USERS"
-            PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
-            echo "$PATCH_JSON"
-            oc patch PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} --type=merge -p "${PATCH_JSON}"
-
-            # wait for sometime as it takes time to create the user, query the secret and check if it is created, otherwise wait in a loop for 10 rounds
-            for i in {1..10}; do
-              if oc get secret postgres-crunchy-${{ inputs.cluster_environment }}-pguser-app-${{ github.event.number }} -o jsonpath='{.metadata.name}' > /dev/null; then
-                echo "Secret created"
-                break
-              else
-                echo "Secret not created, waiting for 60 seconds"
-                sleep 60
-              fi
-            done
+          # check if current_users already contains the new_user
+          if echo "${CURRENT_USERS}" | jq -e ".[] | select(.name == \"app-${{ github.event.number }}\")" > /dev/null; then
+            echo "User already exists"
+            exit 0
+          fi
 
-            # Add public schema and grant to PR user
-            # get primary crunchy pod and remove the role and db
-            CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
-            echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "CREATE SCHEMA IF NOT EXISTS public;"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON SCHEMA public TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO \"app-${{ github.event.number }}\";"
-            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO \"app-${{ github.event.number }}\";"
-            # TODO: remove these
+          UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson NEW_USER "$NEW_USER" '. + [$NEW_USER]')
+          echo "$UPDATED_USERS"
+          PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
+          echo "$PATCH_JSON"
+          oc patch PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} --type=merge -p "${PATCH_JSON}"
 
-            
+          # wait for sometime as it takes time to create the user, query the secret and check if it is created, otherwise wait in a loop for 10 rounds
+          for i in {1..10}; do
+            if oc get secret postgres-crunchy-${{ inputs.cluster_environment }}-pguser-app-${{ github.event.number }} -o jsonpath='{.metadata.name}' > /dev/null; then
+              echo "Secret created"
+              break
+            else
+              echo "Secret not created, waiting for 60 seconds"
+              sleep 60
+            fi
+          done
 
+          # Add public schema and grant to PR user
+          # get primary crunchy pod and remove the role and db
+          CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
+          echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "CREATE SCHEMA IF NOT EXISTS public;"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON SCHEMA public TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO \"app-${{ github.event.number }}\";"
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO \"app-${{ github.event.number }}\";"
+          # TODO: remove these

.github/workflows/analysis.yml

@@ -40,7 +40,7 @@ jobs:
           - dir: frontend
             token: SONAR_TOKEN_FRONTEND
     steps:
-      - uses: bcgov-nr/[email protected]
+      - uses: bcgov/[email protected]
         with:
           commands: |
             npm ci

.github/workflows/merge-hotfix.yml

@@ -32,7 +32,7 @@ jobs:
       # Get PR number for squash merges to main
       - name: PR Number
         id: pr
-        uses: bcgov-nr/[email protected]
+        uses: bcgov/[email protected]
 
   # https://github.com/bcgov/quickstart-openshift-helpers
   deploy-hotfix:

.github/workflows/pr-close.yml

@@ -28,7 +28,7 @@ jobs:
       oc_token: ${{ secrets.OC_TOKEN }}
     with:
       cleanup: label
-  
+
   cleanup-pvcs:
     name: Cleanup Project PVCs
     runs-on: ubuntu-22.04
@@ -60,32 +60,32 @@ jobs:
           oc project ${{ secrets.OC_NAMESPACE }} # Safeguard!
       - run: |
           # check if postgres-crunchy exists or else exit
-          oc get PostgresCluster/postgres-crunchy || exit 0
+          oc get PostgresCluster/postgres-crunchy-dev || exit 0
           # Remove the user from the crunchy cluster yaml and apply the changes
           USER_TO_REMOVE='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
-          
+
           echo 'getting current users from crunchy'
-          CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy -o json | jq '.spec.users')
+          CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy-dev -o json | jq '.spec.users')
           echo "${CURRENT_USERS}"
-          
+
           # Remove the user from the list, 
           UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson user "$USER_TO_REMOVE" 'map(select(. != $user))')
 
           PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
-          oc patch PostgresCluster/postgres-crunchy --type=merge -p "$PATCH_JSON"
-          
+          oc patch PostgresCluster/postgres-crunchy-dev --type=merge -p "$PATCH_JSON"
+
           # get primary crunchy pod and remove the role and db
           CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
-          
+
           echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
           # Terminate all connections to the database before trying terminate
           oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'app-${{ github.event.number }}' AND pid <> pg_backend_pid();"
-          
+
           # Drop the database and role
           oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP DATABASE \"app-${{ github.event.number }}\" --cascade" 
-          
+
           oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP ROLE \"app-${{ github.event.number }}\" --cascade" 
-          
+
           echo "Database and Role for PR is cleaned."
-          
-          exit 0
+
+          exit 0

.github/workflows/pr-open.yml

@@ -16,7 +16,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # https://github.com/bcgov-nr/action-builder-ghcr
+  # https://github.com/bcgov/action-builder-ghcr
   builds:
     name: Builds
     runs-on: ubuntu-22.04
@@ -25,7 +25,7 @@ jobs:
         package: [backend, frontend, migrations, webeoc]
     timeout-minutes: 10
     steps:
-      - uses: bcgov-nr/[email protected]
+      - uses: bcgov/[email protected]
         with:
           package: ${{ matrix.package }}
           tag: ${{ github.event.number }}

backend/dataloader/bulk-data-loader.js

@@ -248,16 +248,69 @@ const insertData = async (data) => {
   }
 };
 
-// Adjust these as required.
-// No more than 10k at a time or the insert will blow up.
-const yearPrefix = 25;
-const numRecords = 10000;
-const startingRecord = 100000;
-
-// Validate parameters
-if (numRecords > 10000) {
-  console.log ("Please adjust the numRecords parameter to be less than 10000");
-  return;
-}
-const records = generateBulkData(yearPrefix, numRecords, startingRecord);
-insertData(records);
+// Process any pending complaints
+const processPendingComplaints = async () => {
+  const query = `DO $$
+    DECLARE
+      complaint_record RECORD;
+    BEGIN
+      -- Loop through each pending complaint
+      FOR complaint_record IN
+          SELECT complaint_identifier
+          FROM staging_complaint
+          WHERE staging_status_code = 'PENDING'
+      LOOP
+          -- Call the insert_complaint_from_staging function for each complaint
+          PERFORM public.insert_complaint_from_staging(complaint_record.complaint_identifier);
+      END LOOP;
+    
+      TRUNCATE TABLE staging_complaint;
+
+      -- Optional: Add a message indicating completion
+      RAISE NOTICE 'All pending complaints processed successfully.';
+    END;
+    $$;`;
+
+  try {
+    // Perform the bulk insert
+    await pg.none(query);
+    console.log('Data processing complete.');
+  } catch (error) {
+    console.error('Error processing data:', error);
+  }
+};
+
+const bulkDataLoad = async () => {
+
+  // Adjust these as required.
+  const processAfterInsert = true // Will move complaints from staging to the live table after each iteration
+  const numRecords = 10000; // Records created per iteration, no more than 10k at a time or the insert will blow up.
+  const yearPrefix = 20; // Year will increment per iteration
+  const startingRecord = 110000;
+  const iterations = 10;
+
+  // Validate parameters
+  if (numRecords > 10000) {
+    console.log ("Please adjust the numRecords parameter to be less than 10000");
+    return;
+  }
+  if (iterations * numRecords > 1000000) {
+    console.log ("Please adjust the iterations so that fewer than 1000000 records are inserted");
+    return;
+  }
+
+  for (let i = 0; i < iterations; i++) {
+    const startingRecordForIteration = startingRecord + (i * numRecords);
+    const records = generateBulkData(yearPrefix + i, numRecords, startingRecordForIteration);
+    // Insert the staging data
+    await insertData(records);
+    // Process the staging data
+    if (processAfterInsert)
+    {
+      await processPendingComplaints();
+    }
+    console.log(`Inserted records ${startingRecordForIteration} through ${startingRecordForIteration + numRecords}.`);
+  }
+};
+
+bulkDataLoad();