Twins importer

charts/twins-api/templates/_helpers.tpl

@@ -10,6 +10,10 @@
 {{ include "twins.name" . }}-migrate
 {{- end }}
 
+{{- define "twins.importer.name" -}}
+{{ include "twins.name" . }}-importer
+{{- end }}
+
 {{- define "twins.secret.deploys.name" -}}
 {{ include "twins.name" . }}-secret-deploys
 {{- end }}
@@ -44,15 +48,35 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 
+{{- define "twins.importer.labels" -}}
+app.kubernetes.io/name: {{ .Chart.Name }}-importer
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+
+{{- define "twins.manifestCode" -}}
+{{- base $.Values.dgctlStorage.manifest | trimSuffix ".json" }}
+{{- end }}
+
+{{- define "twins.env.loglevel" -}}
+- name: TWINS_LOG_LEVEL
+  value: "{{ .Values.api.logLevel }}"
+{{- end }}
+
 {{- define "twins.env.db" -}}
 - name: TWINS_DB_RO_HOST
   value: "{{ required "A valid .Values.postgres.ro.host required" .Values.postgres.ro.host }}"
 - name: TWINS_DB_RO_PORT
   value: "{{ .Values.postgres.ro.port }}"
 - name: TWINS_DB_RO_NAME
   value: "{{ required "A valid .Values.postgres.ro.name required" .Values.postgres.ro.name }}"
+{{- if .Values.importer.postgres.schemaSwitchEnabled }}
+- name: TWINS_DB_RO_SCHEMA
+  value: "{{ include "twins.manifestCode" . }}"
+{{- else }}
 - name: TWINS_DB_RO_SCHEMA
   value: "{{ .Values.postgres.ro.schema }}"
+{{- end }}
 - name: TWINS_DB_RO_CONNECTION_TIMEOUT
   value: "{{ .Values.postgres.ro.timeout }}"
 - name: TWINS_DB_RO_USERNAME
@@ -61,8 +85,13 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
   value: "{{ required "A valid .Values.postgres.rw.host required" .Values.postgres.rw.host }}"
 - name: TWINS_DB_RW_PORT
   value: "{{ .Values.postgres.rw.port }}"
+{{- if .Values.importer.postgres.schemaSwitchEnabled }}
+- name: TWINS_DB_RW_SCHEMA
+  value: "{{ include "twins.manifestCode" . }}"
+{{- else }}
 - name: TWINS_DB_RW_SCHEMA
   value: "{{ .Values.postgres.rw.schema }}"
+{{- end }}
 - name: TWINS_DB_RW_CONNECTION_TIMEOUT
   value: "{{ .Values.postgres.rw.timeout }}"
 - name: TWINS_DB_RW_NAME
@@ -87,6 +116,7 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 
 {{- define "twins.env.db.jobs" -}}
 {{ include "twins.env.db" . }}
+{{ include "twins.env.loglevel" . }}
 - name: TWINS_DB_RO_PASSWORD
   valueFrom:
     secretKeyRef:
@@ -100,6 +130,7 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 
 {{- define "twins.env.api"}}
+{{ include "twins.env.loglevel" . }}
 {{ include "twins.env.db.deploys" . }}
 - name: TWINS_AUTH_ENDPOINT
   value: "{{ required "A valid .Values.api.keys.url required" .Values.api.keys.url }}"
@@ -112,6 +143,38 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
   value: "{{ .Values.api.keys.requestTimeout }}"
 {{- end }}
 
+{{- define "twins.env.importer" -}}
+{{ include "twins.env.db.jobs" . }}
+- name: TWINS_IMPORTER_DB_SCHEMA_SWITCH_ENABLED
+  value: "{{ .Values.importer.postgres.schemaSwitchEnabled }}"
+- name: TWINS_S3_ENDPOINT
+  value: "{{ .Values.dgctlStorage.host }}"
+- name: TWINS_S3_BUCKET
+  value: "{{ .Values.dgctlStorage.bucket }}"
+- name: TWINS_S3_ACCESS_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "twins.secret.jobs.name" . }}
+      key: dgctlStorageAccessKey
+- name: TWINS_S3_SECRET_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "twins.secret.jobs.name" . }}
+      key: dgctlStorageSecretKey
+- name: TWINS_IMPORTER_MANIFEST_PATH
+  value: "{{ required "A valid .Values.dgctlStorage.manifest entry required" .Values.dgctlStorage.manifest }}"
+- name: TWINS_IMPORTER_NUMBER_SCHEMA_BACKUPS
+  value: "{{ .Values.importer.cleaner.versionLimit }}"
+- name: TWINS_S3_RETRY_MAX_ATTEMPTS
+  value: "{{ .Values.importer.retry.download.maxAttempts }}"
+- name: TWINS_S3_RETRY_DELAY
+  value: "{{ .Values.importer.retry.download.delay }}"
+- name: TWINS_IMPORTER_PSQL_RETRY_MAX_ATTEMPTS
+  value: "{{ .Values.importer.retry.execute.maxAttempts }}"
+- name: TWINS_IMPORTER_PSQL_RETRY_DELAY
+  value: "{{ .Values.importer.retry.execute.delay }}"
+{{- end }}
+
 {{/*
 Return the target Kubernetes version
 */}}

charts/twins-api/templates/importer/cleaner/job.yaml

@@ -0,0 +1,33 @@
+{{- if and .Values.importer.cleaner.enabled .Values.importer.postgres.schemaSwitchEnabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "twins.importer.name" . }}-cleaner
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: {{ include "twins.importer.name" . }}-cleaner
+      labels:
+        {{- include "twins.importer.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: cleaner-job
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          command: [ "twins-importer", "clear" ]
+          resources:
+            {{- toYaml .Values.importer.cleaner.resources | nindent 12 }}
+          env:
+            {{- include "twins.env.importer" . | nindent 12 }}
+      {{- with .Values.importer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/twins-api/templates/importer/job.yaml

@@ -0,0 +1,53 @@
+{{- if .Values.importer.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "twins.importer.name" . }}
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "10"
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: {{ include "twins.importer.name" . }}
+      labels:
+        {{- include "twins.importer.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      {{- if .Values.importer.initialDelaySeconds }}
+      initContainers:
+        - name: delay
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          command: [ "sh", "-c", "sleep {{ .Values.importer.initialDelaySeconds }}" ]
+          resources:
+            {{- toYaml .Values.importer.resources | nindent 12 }}
+      {{- end }}
+      containers:
+        - name: importer
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          command: [ "twins-importer", "import" ]
+          resources:
+            {{- toYaml .Values.importer.resources | nindent 12 }}
+          env:
+            {{- include "twins.env.importer" . | nindent 12 }}
+          {{- if .Values.importer.persistentVolume.enabled }}
+          volumeMounts:
+            - name: {{ include "twins.importer.name" . }}-pv
+              mountPath: "/tmp"
+          {{- end }}
+      {{- if .Values.importer.persistentVolume.enabled }}
+      volumes:
+        - name: {{ include "twins.importer.name" . }}-pv
+          persistentVolumeClaim:
+            claimName: {{ include "twins.importer.name" . }}
+      {{- end }}
+      {{- with .Values.importer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/twins-api/templates/importer/pvc.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.importer.persistentVolume.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "twins.importer.name" . }}
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "-20"
+spec:
+  storageClassName: {{ .Values.importer.persistentVolume.storageClass }}
+  {{- with .Values.importer.persistentVolume.accessModes }}
+  accessModes:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.importer.persistentVolume.size }}
+{{- end }}

charts/twins-api/templates/migrate/job.yaml

@@ -1,7 +1,8 @@
+{{- if not .Values.importer.enabled }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "twins.migrate.name" . }}-migrate
+  name: {{ include "twins.migrate.name" . }}
   labels:
     {{- include "twins.migrate.labels" . | nindent 4 }}
   annotations:
@@ -38,3 +39,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}

charts/twins-api/templates/secret-post.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "twins.secret.jobs.name" . }}
+  labels:
+    {{- include "twins.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "-20"
+type: Opaque
+data:
+  dbROPassword:  {{ required "A valid .Values.postgres.ro.password required" .Values.postgres.ro.password | b64enc }}
+  dbRWPassword:  {{ required "A valid .Values.postgres.rw.password required" .Values.postgres.rw.password | b64enc }}
+  dgctlStorageAccessKey:  {{ required "A valid .Values.dgctlStorage.accessKey required" .Values.dgctlStorage.accessKey | b64enc }}
+  dgctlStorageSecretKey:  {{ required "A valid .Values.dgctlStorage.secretKey required" .Values.dgctlStorage.secretKey | b64enc }}

charts/twins-api/templates/secret-jobs.yaml → charts/twins-api/templates/secret-pre.yaml

@@ -12,3 +12,5 @@ type: Opaque
 data:
   dbROPassword:  {{ required "A valid .Values.postgres.ro.password required" .Values.postgres.ro.password | b64enc }}
   dbRWPassword:  {{ required "A valid .Values.postgres.rw.password required" .Values.postgres.rw.password | b64enc }}
+  dgctlStorageAccessKey:  {{ required "A valid .Values.dgctlStorage.accessKey required" .Values.dgctlStorage.accessKey | b64enc }}
+  dgctlStorageSecretKey:  {{ required "A valid .Values.dgctlStorage.secretKey required" .Values.dgctlStorage.secretKey | b64enc }}