Twins importer

charts/twins-api/templates/_helpers.tpl

@@ -10,6 +10,10 @@
 {{ include "twins.name" . }}-migrate
 {{- end }}
 
+{{- define "twins.importer.name" -}}
+{{ include "twins.name" . }}-importer
+{{- end }}
+
 {{- define "twins.secret.deploys.name" -}}
 {{ include "twins.name" . }}-secret-deploys
 {{- end }}
@@ -44,15 +48,30 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 
+{{- define "twins.importer.labels" -}}
+app.kubernetes.io/name: {{ .Chart.Name }}-importer
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+
+{{- define "twins.manifestCode" -}}
+{{- base $.Values.dgctlStorage.manifest | trimSuffix ".json" }}
+{{- end }}
+
 {{- define "twins.env.db" -}}
 - name: TWINS_DB_RO_HOST
   value: "{{ required "A valid .Values.postgres.ro.host required" .Values.postgres.ro.host }}"
 - name: TWINS_DB_RO_PORT
   value: "{{ .Values.postgres.ro.port }}"
 - name: TWINS_DB_RO_NAME
   value: "{{ required "A valid .Values.postgres.ro.name required" .Values.postgres.ro.name }}"
+{{- if .Values.importer.postgres.schemaSwitchEnabled }}
+- name: TWINS_DB_RO_SCHEMA
+  value: "{{ include "twins.manifestCode" . }}"
+{{- else -}}
 - name: TWINS_DB_RO_SCHEMA
   value: "{{ .Values.postgres.ro.schema }}"
+{{- end }}
 - name: TWINS_DB_RO_CONNECTION_TIMEOUT
   value: "{{ .Values.postgres.ro.timeout }}"
 - name: TWINS_DB_RO_USERNAME
@@ -61,8 +80,13 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
   value: "{{ required "A valid .Values.postgres.rw.host required" .Values.postgres.rw.host }}"
 - name: TWINS_DB_RW_PORT
   value: "{{ .Values.postgres.rw.port }}"
+{{- if .Values.importer.postgres.schemaSwitchEnabled }}
+- name: TWINS_DB_RW_SCHEMA
+  value: "{{ include "twins.manifestCode" . }}"
+{{- else -}}
 - name: TWINS_DB_RW_SCHEMA
   value: "{{ .Values.postgres.rw.schema }}"
+{{- end }}
 - name: TWINS_DB_RW_CONNECTION_TIMEOUT
   value: "{{ .Values.postgres.rw.timeout }}"
 - name: TWINS_DB_RW_NAME
@@ -112,6 +136,38 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
   value: "{{ .Values.api.keys.requestTimeout }}"
 {{- end }}
 
+{{- define "twins.env.importer" -}}
+{{ include "twins.env.db.jobs" . }}
+- name: TWINS_IMPORTER_DB_SCHEMA_SWITCH_ENABLED
+  value: "{{ .Values.importer.postgres.schemaSwitchEnabled }}"
+- name: TWINS_S3_ENDPOINT
+  value: "{{ .Values.dgctlStorage.host }}"
+- name: TWINS_S3_BUCKET
+  value: "{{ .Values.dgctlStorage.bucket }}"
+- name: TWINS_S3_ACCESS_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "twins.secret.jobs.name" . }}
+      key: dgctlStorageAccessKey
+- name: TWINS_S3_SECRET_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "twins.secret.jobs.name" . }}
+      key: dgctlStorageSecretKey
+- name: TWINS_IMPORTER_MANIFEST_PATH
+  value: "{{ required "A valid .Values.dgctlStorage.manifest entry required" .Values.dgctlStorage.manifest }}"
+- name: TWINS_IMPORTER_NUMBER_SCHEMA_BACKUPS
+  value: "{{ .Values.importer.cleaner.versionLimit }}"
+- name: TWINS_S3_RETRY_MAX_ATTEMPTS
+  value: "{{ .Values.importer.retry.download.maxAttempts }}"
+- name: TWINS_S3_RETRY_DELAY
+  value: "{{ .Values.importer.retry.download.delay }}"
+- name: TWINS_IMPORTER_PSQL_RETRY_MAX_ATTEMPTS
+  value: "{{ .Values.importer.retry.execute.maxAttempts }}"
+- name: TWINS_IMPORTER_PSQL_RETRY_DELAY
+  value: "{{ .Values.importer.retry.execute.delay }}"
+{{- end }}
+
 {{/*
 Return the target Kubernetes version
 */}}

charts/twins-api/templates/importer/cleaner/job.yaml

@@ -0,0 +1,33 @@
+{{- if and .Values.importer.cleaner.enabled .Values.importer.postgres.schemaSwitchEnabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "twins.importer.name" . }}-cleaner
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: {{ include "twins.importer.name" . }}-cleaner
+      labels:
+        {{- include "twins.importer.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: cleaner-job
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          command: [ "twins-importer", "clear" ]
+          resources:
+            {{- toYaml .Values.importer.cleaner.resources | nindent 12 }}
+          env:
+            {{- include "twins.env.importer" . | nindent 12 }}
+      {{- with .Values.importer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/twins-api/templates/importer/job.yaml

@@ -0,0 +1,53 @@
+{{- if .Values.importer.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "twins.importer.name" . }}
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "10"
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: {{ include "twins.importer.name" . }}
+      labels:
+        {{- include "twins.importer.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      {{- if .Values.importer.initialDelaySeconds }}
+      initContainers:
+        - name: delay
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          command: [ "sh", "-c", "sleep {{ .Values.importer.initialDelaySeconds }}" ]
+          resources:
+            {{- toYaml .Values.importer.resources | nindent 12 }}
+      {{- end }}
+      containers:
+        - name: importer
+          image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          command: [ "twins-importer", "import" ]
+          resources:
+            {{- toYaml .Values.importer.resources | nindent 12 }}
+          env:
+            {{- include "twins.env.importer" . | nindent 12 }}
+          {{- if .Values.importer.persistentVolume.enabled }}
+          volumeMounts:
+            - name: {{ include "twins.importer.name" . }}-pv
+              mountPath: "/tmp"
+          {{- end }}
+      {{- if .Values.importer.persistentVolume.enabled }}
+      volumes:
+        - name: {{ include "twins.importer.name" . }}-pv
+          persistentVolumeClaim:
+            claimName: {{ include "twins.importer.name" . }}
+      {{- end }}
+      {{- with .Values.importer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/twins-api/templates/importer/pvc.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.importer.persistentVolume.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "twins.importer.name" . }}
+  labels:
+    {{- include "twins.importer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "-20"
+spec:
+  storageClassName: {{ .Values.importer.persistentVolume.storageClass }}
+  {{- with .Values.importer.persistentVolume.accessModes }}
+  accessModes:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.importer.persistentVolume.size }}
+{{- end }}

charts/twins-api/templates/migrate/job.yaml

@@ -1,7 +1,8 @@
+{{- if not .Values.importer.enabled }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "twins.migrate.name" . }}-migrate
+  name: {{ include "twins.migrate.name" . }}
   labels:
     {{- include "twins.migrate.labels" . | nindent 4 }}
   annotations:
@@ -38,3 +39,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}

charts/twins-api/templates/secret-post.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "twins.secret.jobs.name" . }}
+  labels:
+    {{- include "twins.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-weight": "-20"
+type: Opaque
+data:
+  dbROPassword:  {{ required "A valid .Values.postgres.ro.password required" .Values.postgres.ro.password | b64enc }}
+  dbRWPassword:  {{ required "A valid .Values.postgres.rw.password required" .Values.postgres.rw.password | b64enc }}
+  dgctlStorageAccessKey:  {{ required "A valid .Values.dgctlStorage.accessKey required" .Values.dgctlStorage.accessKey | b64enc }}
+  dgctlStorageSecretKey:  {{ required "A valid .Values.dgctlStorage.secretKey required" .Values.dgctlStorage.secretKey | b64enc }}

charts/twins-api/templates/secret-jobs.yaml → charts/twins-api/templates/secret-pre.yaml

@@ -12,3 +12,5 @@ type: Opaque
 data:
   dbROPassword:  {{ required "A valid .Values.postgres.ro.password required" .Values.postgres.ro.password | b64enc }}
   dbRWPassword:  {{ required "A valid .Values.postgres.rw.password required" .Values.postgres.rw.password | b64enc }}
+  dgctlStorageAccessKey:  {{ required "A valid .Values.dgctlStorage.accessKey required" .Values.dgctlStorage.accessKey | b64enc }}
+  dgctlStorageSecretKey:  {{ required "A valid .Values.dgctlStorage.secretKey required" .Values.dgctlStorage.secretKey | b64enc }}

charts/twins-api/values.yaml

@@ -14,6 +14,21 @@ dgctlDockerRegistry: ''
 
 imagePullSecrets: []
 
+# @section Deployment Artifacts Storage settings
+
+# @param dgctlStorage.host S3 host. Format: `host:port`. **Required**
+# @param dgctlStorage.bucket S3 bucket name. **Required**
+# @param dgctlStorage.accessKey S3 access key for accessing the bucket. **Required**
+# @param dgctlStorage.secretKey S3 secret key for accessing the bucket. **Required**
+# @param dgctlStorage.manifest The path to the [manifest file](https://docs.2gis.com/en/on-premise/overview#nav-lvl2@paramCommon_deployment_steps). Format: `manifests/0000000000.json` <br> This file contains the description of pieces of data that the service requires to operate. **Required**
+
+dgctlStorage:
+  host: ''
+  bucket: ''
+  accessKey: ''
+  secretKey: ''
+  manifest: ''
+
 image:
   repository: 2gis-on-premise/twins-api
   tag: 1.6.0
@@ -45,6 +60,12 @@ api:
 
   replicas: 1
 
+  # @section api.resources **Kubernetes [resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) settings**
+  # @param api.resources.requests.cpu A CPU request.
+  # @param api.resources.requests.memory A memory request.
+  # @param api.resources.limits.cpu A CPU limit.
+  # @param api.resources.limits.memory A memory limit.
+
   resources:
     requests:
       cpu: 50m
@@ -131,6 +152,12 @@ migrate:
 
   initialDelaySeconds: 0
 
+  # @section migrate.resources **Kubernetes [resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) settings**
+  # @param migrate.resources.requests.cpu A CPU request.
+  # @param migrate.resources.requests.memory A memory request.
+  # @param migrate.resources.limits.cpu A CPU limit.
+  # @param migrate.resources.limits.memory A memory limit.
+
   resources:
     requests:
       cpu: 10m
@@ -187,16 +214,82 @@ postgres:
     password: ''
 
 
-# @section Limits
+# @section Kubernetes Importer job settings
+
+# @extra importer **Common settings**
+# @param importer.enabled If importer is enabled for the service
+# @param importer.nodeSelector Kubernetes [node selectors](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)
+# @param importer.initialDelaySeconds Number of seconds after the container has started before liveness or readiness probes are initiated
+# @param importer.retry.download.maxAttempts The maximum number of retries download before stopping
+# @param importer.retry.download.delay Delay until the retry attempts download
+# @param importer.retry.execute.maxAttempts The maximum number of retries execute psql command before stopping
+# @param importer.retry.execute.delay Delay until the retry attempts execute
+
+importer:
+  enabled: true
+  nodeSelector: {}
+  initialDelaySeconds: 1
+  retry:
+    download:
+      maxAttempts: 3
+      delay: 1s
+    execute:
+      maxAttempts: 3
+      delay: 1s
+
+  # @section importer.postgres **Database settings**
+  # @param importer.postgres.schemaSwitchEnabled Automatic switch PostgreSQL schema on releases
 
-# @extra api.resources **Limits for the API service**
-# @param api.resources.requests.cpu A CPU request.
-# @param api.resources.requests.memory A memory request.
-# @param api.resources.limits.cpu A CPU limit.
-# @param api.resources.limits.memory A memory limit.
+  postgres:
+    schemaSwitchEnabled: true
 
-# @extra migrate.resources **Limits for the Migrate service**
-# @param migrate.resources.requests.cpu A CPU request.
-# @param migrate.resources.requests.memory A memory request.
-# @param migrate.resources.limits.cpu A CPU limit.
-# @param migrate.resources.limits.memory A memory limit.
+  # @section importer.persistentVolume **Persistent Volume settings**
+
+  # @param importer.persistentVolume.enabled If [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) is enabled for the service
+  # @param importer.persistentVolume.accessModes Persistent Volume [Access Mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
+  # @param importer.persistentVolume.storageClass Kubernetes [Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/)
+  # @param importer.persistentVolume.size Volume size
+
+  persistentVolume:
+    enabled: false
+    accessModes:
+      - ReadWriteOnce
+    storageClass: topolvm-ext4
+    size: 50Gi
+
+  # @section importer.resources **Kubernetes [resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) settings**
+  # @param importer.resources.requests.cpu A CPU request
+  # @param importer.resources.requests.memory A memory request
+  # @param importer.resources.limits.cpu A CPU limit
+  # @param importer.resources.limits.memory A memory limit
+
+  resources:
+    requests:
+      cpu: 256m
+      memory: 512Mi
+    limits:
+      cpu: 2
+      memory: 2048Mi
+
+  # @section importer.cleaner **Cleaner scheme settings**
+  # @param importer.cleaner.enabled If clean schemes is enabled for the service
+  # @param importer.cleaner.versionLimit Number of backup schemes
+
+
+  cleaner:
+    enabled: true
+    versionLimit: 2
+
+    # @section importer.cleaner.resources **Kubernetes [resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) settings**
+    # @param importer.cleaner.resources.requests.cpu A CPU request
+    # @param importer.cleaner.resources.requests.memory A memory request
+    # @param importer.cleaner.resources.limits.cpu A CPU limit
+    # @param importer.cleaner.resources.limits.memory A memory limit
+
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        cpu: 1000m
+        memory: 512Mi