feat(executor): Add fluent-bit sidecar (WIP don't merge)

acryldata · Dec 19, 2024 · ff17e64 · ff17e64
1 parent 921e168
commit ff17e64
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 1 deletion.
diff --git a/charts/datahub-executor-worker/templates/configmap.yaml b/charts/datahub-executor-worker/templates/configmap.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+{{- if .Values.extraConfigmap }}
+    name: {{ .Values.extraConfigmap.name }}
+{{- end }}
+
+data:
+{{- if .Values.extraConfigmap.data }}
+  {{- range .Values.extraConfigmap.data }}
+  {{- range $key, $value := . }}
+  {{ $key }}: |
+    {{ $value | nindent 4 }}
+  {{- end }}
+ {{- end }}
+{{- end }}
diff --git a/charts/datahub-executor-worker/templates/workload.yaml b/charts/datahub-executor-worker/templates/workload.yaml
@@ -163,7 +163,24 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-
+      {{- if .Values.sidecarContainers }}
+        {{- range $name, $spec :=  .Values.sidecarContainers }}
+        - name: {{ $name }}
+          {{- if kindIs "string" $spec }}
+            {{- tpl $spec $ | nindent 10 }}
+          {{- else }}
+            {{- toYaml $spec | nindent 10 }}
+          {{- end }}
+        {{- end }}
+      volumes:
+         {{- if .Values.extraConfigmapMounts }}
+          {{- range .Values.extraConfigmapMounts }}
+             - name: {{ .name }}
+               configMap:
+                 name: {{ .configMap }}
+             {{- end }}
+           {{- end }}
+         {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/charts/datahub-executor-worker/values.yaml b/charts/datahub-executor-worker/values.yaml
@@ -24,6 +24,31 @@ image:
   pullPolicy: Always
   tag: v0.3.7.3-acryl
 
+sidecarContainers:
+  sidecar-fluentbit:
+    image: "cr.fluentbit.io/fluent/fluent-bit:3.2.2"
+    imagePullPolicy: IfNotPresent
+    command:
+       - /fluent-bit/bin/fluent-bit
+    args:
+      - --workdir=/fluent-bit/etc
+      - --config=/fluent-bit/etc/conf/fluent-bit.conf
+    ports:
+      - name: http
+        containerPort: 2021
+        protocol: TCP
+    livenessProbe:
+      httpGet:
+        path: /
+        port: http
+    readinessProbe:
+      httpGet:
+        path: /api/v1/health
+        port: http
+    volumeMounts:
+      - name: fluentbit-config
+        mountPath: /fluent-bit/etc/conf
+
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
@@ -61,6 +86,41 @@ extraEnvs: []
 
 extraVolumes: []
 
+extraConfigmapMounts:
+ - name: fluentbit-config
+   mountPath: /fluent-bit/etc/conf
+   configMap: sidecar-fluent-bit
+   readOnly: true
+
+extraConfigmap:
+  name: sidecar-fluent-bit
+  data:
+    - fluent-bit.conf: |
+        [SERVICE]
+            flush           1
+            daemon          Off
+            log_level       info
+            parsers_file    parsers.conf
+            plugins_file    plugins.conf
+            http_server     On
+            http_listen     0.0.0.0
+            http_port       2021
+            storage.metrics on
+        [INPUT]
+            name            prometheus_scrape
+            host            127.0.0.1
+            port            9087
+            tag             remote
+            scrape_interval 60
+        [OUTPUT]
+            Name            prometheus_remote_write
+            Match           *
+            Host            prometheus-operated
+            Port            9090
+            Uri             /api/v1/write?prometheus_server=remote
+            add_label       source remote-executor
+
+
 extraVolumeMounts: []
 
 extraInitContainers: []