(Cypress) Fix orakl-db-backup (#245)

* fix: Remove sudo command * feat: Remove job when it's successful * fix: Correct indent * feat: Execute whole command into single item * chore: Add escape char * chore: Remove escape char * chore: Add escape char again * fix: Remove escape char and pass argument directly * chore: test * test * test * test * test * test * test * feat: Update dependencies * feat: Add drop table --------- Co-authored-by: jay-bisonai <[email protected]>
Bisonai · Mar 18, 2024 · 7aa5eed · 7aa5eed
1 parent 2e24c21
commit 7aa5eed
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 9 deletions.
diff --git a/cronjobs/templates/cronjob.yaml b/cronjobs/templates/cronjob.yaml
@@ -12,6 +12,7 @@ spec:
   failedJobsHistoryLimit: {{ $job.failedJobsHistoryLimit }}
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: {{ $job.ttlSecondsAfterFinished }}
       template:
         metadata:
           labels:

diff --git a/cronjobs/values.yaml b/cronjobs/values.yaml
@@ -11,23 +11,31 @@ jobs:
     failedJobHistoryLimit: 3
     successfulJobHistoryLimit: 10
     concurrencyPolicy: Allow
-    restartPolicy: OnFailure
+    restartPolicy: Never 
+    ttlSecondsAfterFinished: 30
     serviceAccount:
       annotations:
         iam.gke.io/gcp-serviceaccount: "[email protected]"
     command: ["/bin/bash"]
     args:
       - "-c"
-      # Step 1: Install postgresql-client
-      - "sudo apt -y update && sudo apt -y install postgresql-client"
-      # Step 2: Export old data to csv file.
       - |
+        echo "Step 1: Install dependencies"
+        apt -y update && apt -y install curl apt-transport-https ca-certificates gnupg postgresql-client
+        echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg |  gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && apt-get update -y && apt-get install google-cloud-sdk -y
+        mkdir -p /data/csv
+        chmod -R 777 /data/csv
+
+        echo "Step 2: Export old data to csv files"
         psql $DATABASE_URL -t -c "SELECT tablename FROM pg_tables WHERE tablename LIKE 'data_%';" \
-        | xargs -I {} sh -c '\
-        psql $DATABASE_URL -c "\copy {} to './{}.csv' WITH DELIMITER ',' CSV HEADER;" \
-        && psql $DATABASE_URL -c "DROP TABLE {};"'
-      # Step 3: Upload all csv files to GCS bucket.
-      - "ls | grep 'csv' | xargs -I {} gcloud storage cp {} gs://orakl-db-backup-cypress"
+        | xargs -I {} psql $DATABASE_URL -c "\copy {} to '/data/csv/{}.csv' WITH DELIMITER ',' CSV HEADER;"
+
+        echo "Step 3: Upload all csv files to GCS bucket."
+        gcloud storage cp /data/csv/*.csv gs://orakl-db-backup-cypress
+
+        echo "Step 4: Delete old data tables"
+        psql $DATABASE_URL -t -c "SELECT tablename FROM pg_tables WHERE tablename LIKE 'data_%';" \
+        | xargs -I {} psql $DATABASE_URL -c "DROP TABLE ${};"
     env:
       - name: DATABASE_URL
         valueFrom: