add rudderstack 0.5.0

LuxorLabs · Apr 25, 2023 · 481c05b · 481c05b
1 parent 042c8bf
commit 481c05b
Show file tree

Hide file tree

Showing 9 changed files with 102 additions and 33 deletions.
diff --git a/charts/rudderstack-helm/Chart.yaml b/charts/rudderstack-helm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: rudderstack
-description: A Helm chart for Kubernetes
+description: Privacy and Security focused Segment-alternative, in Golang and React
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -14,21 +14,21 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.3.0
+version: 0.5.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 1.16.0
+appVersion: 1.4.3
 
 # WIP
-#dependencies: 
-#  - name: nginx-ingress
-#    version: ~1.6.0
-#    repository: https://helm.nginx.com/stable
-#    condition: (optional) A yaml path that resolves to a boolean, used for enabling/disabling charts (e.g. subchart1.enabled )
-#    tags: # (optional)
-#      - Tags can be used to group charts for enabling/disabling together
-#    enabled: (optional) Enabled bool determines if chart should be loaded
-#    import-values: # (optional)
-#      - ImportValues holds the mapping of source values to parent key to be imported. Each item can be a string or pair of child/parent sublist items.
-#    alias: (optional) Alias usable alias to be used for the chart. Useful when you have to add the same chart multiple times
+# dependencies:
+#   - name: nginx-ingress
+#     version: ~1.6.0
+#     repository: https://helm.nginx.com/stable
+#     condition: (optional) A yaml path that resolves to a boolean, used for enabling/disabling charts (e.g. subchart1.enabled )
+#     tags: # (optional)
+#       - Tags can be used to group charts for enabling/disabling together
+#     enabled: (optional) Enabled bool determines if chart should be loaded
+#     import-values: # (optional)
+#       - ImportValues holds the mapping of source values to parent key to be imported. Each item can be a string or pair of child/parent sublist items.
+#     alias: (optional) Alias usable alias to be used for the chart. Useful when you have to add the same chart multiple times
diff --git a/charts/rudderstack-helm/README.md b/charts/rudderstack-helm/README.md
@@ -74,11 +74,13 @@ The following table lists the configurable parameters of the Rudderstack chart a
 | Parameter                           | Description                                                                                         | Default                  |
 | ----------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------ |
 | `rudderWorkspaceToken`              | Workspace token from the dashboard                                                                  | `-`                      |
+| `rudderWorkspaceTokenExistingSecret`    | Secret with workspace token (overrides `rudderWorkspaceToken`)                                                                 | `-`                      |
 | `backend.image.repository`          | Container image repository for the backend                                                          | `rudderlabs/rudder-server`     |
 | `backend.image.version`                 | Container image tag for the backend. [Available versions](https://hub.docker.com/r/rudderlabs/rudder-server/tags)                                                                 | `v0.1.6`                  |
 | `backend.image.pullPolicy`     | Container image pull policy for the backend image                                                   | `Always`           |
-| `transformer.image.repository`      | Container image repository for the transformer                                                      | `rudderlabs/transformer` |
-| `transformer.image.version`             | Container image tag for the transformer. [Available versions](https://hub.docker.com/r/rudderlabs/rudder-transformer/tags)                                                            | `v0.1.2`                  |
+| `backend.config.overrides` | object | `{}` | rudder-server config overrides See [config parameters](https://docs.rudderlabs.com/administrators-guide/config-parameters) for more details |
+| `transformer.image.repository`      | Container image repository for the transformer                                                      | `rudderstack/transformer` |
+| `transformer.image.version`             | Container image tag for the transformer. [Available versions](https://hub.docker.com/r/rudderstack/rudder-transformer/tags)                                                            | `latest`                  |
 | `transformer.image.pullPolicy` | Container image pull policy for the transformer image                                               | `Always`           |
 | `backend.extraEnvVars`              | Extra environments variables to be used by the backend in the deployments                           | `Refer values.yaml file` |
 | `backend.controlPlaneJSON`                   | If `true`, backend will read config from the workspaceConfig.json file  |  `false` |
@@ -92,7 +94,8 @@ $ helm install --name my-release \
 ```
 
 **Note:** Configuration specific to:
-- Backend can be edited in [rudder-config.yaml](https://docs.rudderlabs.com/administrators-guide/config-parameters).
+
+- Backend can be edited in [rudder-config.yaml](https://github.com/rudderlabs/rudderstack-helm/blob/master/rudder-config.yaml). or in values.yaml under `backend.config.overrides`.
 - PostgreSQL can be edited in `pg_hba.conf`, `postgresql.conf`
 
 ## Components

diff --git a/charts/rudderstack-helm/charts/postgresql/values.yaml b/charts/rudderstack-helm/charts/postgresql/values.yaml
@@ -209,11 +209,11 @@ ldap:
   tls: false
 
 ## PostgreSQL service configuration
-# service:
-#   ## PosgresSQL service type
-#   type: ClusterIP
-#   # clusterIP: None
-#   port: 5432
+service:
+  ## PosgresSQL service type
+  type: ClusterIP
+  # clusterIP: None
+  port: 5432
 
   ## Specify the nodePort value for the LoadBalancer and NodePort service types.
   ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport

diff --git a/charts/rudderstack-helm/rudder-config.yaml b/charts/rudderstack-helm/rudder-config.yaml
@@ -35,6 +35,8 @@ Gateway:
     maxTransformerProcess: 64
     maxRetry: 5
     maxRetryTime: 10s
+    sourceListForParsingParams:
+      - shopify
 EventSchemas:
   enableEventSchemasFeature: false
   syncInterval: 240s

diff --git a/charts/rudderstack-helm/templates/_helpers.tpl b/charts/rudderstack-helm/templates/_helpers.tpl
@@ -51,6 +51,13 @@ app.kubernetes.io/name: {{ include "rudderstack.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end -}}
 
+{{/*
+Return secret name to be used based on provided values.
+*/}}
+{{- define "rudderstack.rudderWorkspaceTokenSecretName" -}}
+{{- default (include "rudderstack.fullname" .) .Values.rudderWorkspaceTokenExistingSecret -}}
+{{- end -}}
+
 {{- define "transformer.name" -}}
 {{- printf "%s-%s" (include "rudderstack.name" .) "transformer" -}}
 {{- end -}}
@@ -86,6 +93,27 @@ Return the appropriate apiVersion for statefulset.
 {{- end -}}
 {{- end -}}
 
+{{- define "statsd.enabled" -}}
+{{- if .Values.telegraf_sidecar -}}
+{{- if .Values.telegraf_sidecar.enabled }}
+{{ .Values.telegraf_sidecar.enabled }}
+{{- end -}}
+{{- else -}}
+{{ fail "Telegraf sidecar block doesn't exist in values.yaml ." }}
+{{- end -}}
+{{- end -}}
+
+{{- define "statsd.url" -}}
+{{- if .Values.telegraf_sidecar -}}
+{{- if .Values.telegraf_sidecar.enabled -}}
+{{- printf "localhost:8125" }}
+{{- else -}}
+{{ fail "Sidecar Telegraf is not enabled. We can't deduce statsd url." }}
+{{- end -}}
+{{- else -}}
+{{ fail "Telegraf sidecar block doesn't exist in values.yaml . We can't deduce statsd url." }}
+{{- end -}}
+{{- end -}}
 
 
 {{/*telegraf helper functions */}}

diff --git a/charts/rudderstack-helm/templates/configmap-rudder-server.yaml b/charts/rudderstack-helm/templates/configmap-rudder-server.yaml
@@ -1,3 +1,6 @@
+---
+{{- $baseConfig := .Files.Get "rudder-config.yaml" | fromYaml -}}
+{{- $mergedConfig := mustMergeOverwrite $baseConfig .Values.backend.config.overrides -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -7,7 +10,7 @@ metadata:
     release: {{ .Release.Name | quote }}
 data:
   config.yaml: |-
-{{ .Files.Get "rudder-config.yaml" | indent 4}}
+    {{ $mergedConfig | toYaml | nindent 4 }}
   google-application-credentials.json: |-
 {{.Files.Get "rudder-google-application-credentials.json" | indent 4}}
   {{- if .Values.backend.controlPlaneJSON}}

diff --git a/charts/rudderstack-helm/templates/secret-rudder-token.yaml b/charts/rudderstack-helm/templates/secret-rudder-token.yaml
@@ -0,0 +1,13 @@
+{{- if not .Values.rudderWorkspaceTokenExistingSecret }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "rudderstack.rudderWorkspaceTokenSecretName" . }}
+  labels:
+    {{- include "rudderstack.labels" . | nindent 4 }}
+type: Opaque
+data:
+   rudderWorkspaceToken: {{ required "value for either .Values.rudderWorkspaceTokenExistingSecret or .Values.rudderWorkspaceToken is expected" .Values.rudderWorkspaceToken | b64enc | quote }}
+{{- end }}
+
diff --git a/charts/rudderstack-helm/templates/statefulset.yaml b/charts/rudderstack-helm/templates/statefulset.yaml
@@ -1,3 +1,5 @@
+{{- $baseConfig := .Files.Get "rudder-config.yaml" | fromYaml -}}
+{{- $rudderConfig := mustMergeOverwrite $baseConfig .Values.backend.config.overrides -}}
 apiVersion: {{ include "statefulset.apiVersion" . }}
 kind: StatefulSet
 metadata:
@@ -15,7 +17,10 @@ spec:
       labels:
         {{- include "rudderstack.selectorLabels" . | nindent 8 }}
       annotations:
-        checksum/rudder-config: {{ .Files.Get "rudder-config.yaml" | sha256sum }}
+        {{- if not .Values.rudderWorkspaceTokenExistingSecret }}
+        checksum/rudder-workspace-token: {{ include (print $.Template.BasePath "/secret-rudder-token.yaml") . | sha256sum }}
+        {{- end }}
+        checksum/rudder-config: {{ $rudderConfig | toYaml | sha256sum }}
         checksum/rudder-bigquery-credentials: {{ .Files.Get "bigquery-credentials.json" | sha256sum }}
     spec:
     {{- with .Values.global.imagePullSecrets }}
@@ -27,12 +32,14 @@ spec:
             defaultMode: 420
             name: {{ include "backend.fullname" . }}-config
           name: backend-config-volume
+        {{- if .Values.telegraf_sidecar }}
         {{- if .Values.telegraf_sidecar.enabled}}
         - name: telegraf
           configMap:
             defaultMode: 420
             name: {{ include "telegraf-sidecar.fullname" . }}-config
         {{- end }}
+        {{- end }}
         {{- if .Values.gcpCredentialSecret.enabled }}
         - name: google-application-credentials
           secret:
@@ -71,7 +78,10 @@ spec:
             {{- .Values.backend.extraEnvVars | toYaml | nindent 10 }}
           {{- end }}
           - name: CONFIG_BACKEND_TOKEN
-            value: {{ .Values.rudderWorkspaceToken }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ template "rudderstack.rudderWorkspaceTokenSecretName" . }}
+                key: rudderWorkspaceToken
             {{- if .Values.backend.controlPlaneJSON }}
           - name: RSERVER_BACKEND_CONFIG_CONFIG_FROM_FILE
             value: "{{ .Values.backend.controlPlaneJSON }}"
@@ -103,7 +113,7 @@ spec:
           - name: KUBE_NAMESPACE
             value: {{ .Release.Namespace }}
         command: ["/docker-entrypoint.sh"]
-        args: ["/bin/sh","-c","/wait-for $JOBS_DB_HOST:$(JOBS_DB_PORT) -- /rudder-server"]
+        args: ["/bin/sh","-c","/wait-for $JOBS_DB_HOST:$(JOBS_DB_PORT) {{ if (include "statsd.enabled" .) -}} -- /wait-for -u {{ include "statsd.url" .}} {{- end }} -- /rudder-server"]
         {{- if .Values.telegraf_sidecar.enabled}}
       - name: {{ include "telegraf-sidecar.name" .}}
         image: "{{ .Values.telegraf_sidecar.image.repo }}:{{ .Values.telegraf_sidecar.image.tag }}"

diff --git a/charts/rudderstack-helm/values.yaml b/charts/rudderstack-helm/values.yaml
@@ -7,8 +7,9 @@
 # Following values must be filled in for the deployment to succeed
 
 # Please uncomment below lines and fill values accordingly.
-# Please enter api token obtained from rudder dashboard below
-# rudderWorkspaceToken:
+# Please enter api token obtained from rudder dashboard below or specify existing secret, that contains rudderWorkspaceToken key
+rudderWorkspaceToken: replaceMe
+# rudderWorkspaceTokenExistingSecret:
 
 gcpCredentialSecret:
   enabled: false
@@ -30,7 +31,7 @@ global:
 backend:
   image:
     repository: rudderlabs/rudder-server
-    version: 1-alpine
+    version: latest
     pullPolicy: Always
   controlPlaneJSON: false
   ingress:
@@ -44,7 +45,7 @@ backend:
     annotations:
       ## Refer https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer for more annotations
       service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
-      ## For enabling https on aws, 
+      ## For enabling https on aws,
       ## uncomment below line with acm managed certificate arn and change port value below to 443
       # service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012
     type: LoadBalancer
@@ -68,6 +69,12 @@ backend:
 
   config:
     mountPath: "/etc/rudderstack"
+    # -- rudder-server config overrides
+    # See [config parameters](https://docs.rudderlabs.com/administrators-guide/config-parameters) for more details
+    overrides: {}
+      # enableStats: false
+      # Logger:
+      #  consoleJsonFormat: true
 
   extraEnvVars:
     - name: CONFIG_BACKEND_URL
@@ -84,22 +91,25 @@ backend:
     - name: RUDDER_TMPDIR
       value: "/data/rudderstack"
     - name: JOBS_BACKUP_STORAGE_PROVIDER
-      value: "S3" # object storage provider to store backups eg. S3, GCS, AZURE_BLOB, MINIO
+      value: "S3"  # object storage provider to store backups eg. S3, GCS, AZURE_BLOB, MINIO
     - name: JOBS_BACKUP_BUCKET
       value: "Add your bucket name to store backups of incoming events"
     - name: JOB_STATUS_BACKUP_BUCKET
       value: "Add your bucket name to store backups of status of incoming events"
     - name: GOOGLE_APPLICATION_CREDENTIALS
       value: "/etc/rudderstack/google-application-credentials.json"
     - name: LOG_LEVEL
-      value: "INFO" # eg. DEBUG, ERROR
+      value: "INFO"  # eg. DEBUG, ERROR
+      # DO NOT REMOVE - Mandatory env for Shopify
+    - name: RSERVER_GATEWAY_WEBHOOK_SOURCE_LIST_FOR_PARSING_PARAMS
+      value: Shopify
 
 transformer:
   replicaCount: 1
   service:
     port: 9090
   image:
-    repository: rudderlabs/rudder-transformer
+    repository: rudderstack/rudder-transformer
     version: latest
     pullPolicy: Always
   resources: