Test/eike deploy pr

helm-chart/renku/requirements.yaml

@@ -4,8 +4,8 @@ dependencies:
   version: 0.8.0
   condition: gitlab.enabled
 - name: postgresql
-  version: 9.1.1
-  repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
+  version: 14.0.1
+  repository: "https://charts.bitnami.com/bitnami"
   condition: postgresql.enabled
 - name: keycloakx
   version: 2.1.0
@@ -32,4 +32,7 @@ dependencies:
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"
   version: "0.1.6"
   condition: global.csi-rclone.install
-
+- name: solr
+  repository: "oci://registry-1.docker.io/bitnamicharts"
+  version: "8.9.2"
+  condition: solr.enabled

helm-chart/renku/templates/_helpers.tpl

@@ -57,6 +57,10 @@ Define subcharts full names
 {{- printf "%s-%s" .Release.Name "keycloakx" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{- define "solr.fullname" -}}
+{{- printf "%s-%s" .Release.Name "solr" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "gitlab.fullname" -}}
 {{- printf "%s-%s" .Release.Name "gitlab" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
@@ -164,3 +168,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- define "renku.keycloak.realm" -}}
 {{ .Values.global.keycloak.realm | default "Renku" }}
 {{- end -}}
+
+{{- define "renku.dataService.keycloak.clientId" -}}
+data-service
+{{- end -}}

helm-chart/renku/templates/data-service/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       labels:
         app: renku-data-service
         release: {{ .Release.Name }}
+        {{ .Values.global.redis.clientLabel | toYaml | nindent 8 }}
       {{- with .Values.dataService.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
@@ -58,6 +59,13 @@ spec:
               value: {{ (printf "%s://%s/auth/" (include "renku.http" .) .Values.global.renku.domain) | quote }}
             - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS
               value: "RS256"
+            - name: KEYCLOAK_CLIENT_ID
+              value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+            - name: KEYCLOAK_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "renku.fullname" . }}
+                  key: dataServiceKeycloakClientSecret
             - name: SERVER_DEFAULTS
               value: /etc/renku-data-service/server_options/server_defaults.json
             - name: SERVER_OPTIONS
@@ -68,6 +76,21 @@ spec:
               value: {{ .Values.global.gitlab.url | quote }}
             - name: MAX_PINNED_PROJECTS
               value: {{ .Values.dataService.maxPinnedProjects | quote }}
+            - name: REDIS_HOST
+              value: {{ .Values.global.redis.host | quote }}
+            - name: REDIS_PORT
+              value: {{ .Values.global.redis.port | quote }}
+            - name: REDIS_DATABASE
+              value: {{ .Values.global.redis.dbIndex.events| quote }}
+            - name: REDIS_IS_SENTINEL
+              value: {{ .Values.global.redis.sentinel.enabled | quote }}
+            - name: REDIS_MASTER_SET
+              value: {{ .Values.global.redis.sentinel.masterSet | quote }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.redis.existingSecret }}
+                  key: {{ .Values.global.redis.existingSecretPasswordKey }}
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: server-options

helm-chart/renku/templates/ingress.yaml

@@ -12,6 +12,7 @@
 {{- $graphEnabled := .Values.graph.enabled -}}
 {{- $webhookServiceFullname := include "renku.graph.webhookService.fullname" . -}}
 {{- $knowledgeGraphFullname := include "renku.graph.knowledgeGraph.fullname" . -}}
+{{- $searchApiFullname := include "renku.search.searchApi.fullname" . -}}
 {{- $renkuFullname := include "renku.fullname" . -}}
 {{- if $graphEnabled }}
 {{- $jenaFullname := include "renku-jena.fullname" . -}}
@@ -126,6 +127,13 @@ spec:
             port:
               number: 80
       {{- end }}
+      - path: /apiv2/search
+        pathType: Prefix
+        backend:
+          service:
+            name: {{ $searchApiFullname }}
+            port:
+              number: 80
       {{- if $swaggerEnabled }}
       - path: /swagger
         pathType: Prefix

helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml

@@ -0,0 +1,113 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "renku.fullname" . }}-keycloak-sync-events
+  labels:
+    app: keycloak-sync
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ template "renku.chart" . }}
+spec:
+  schedule: "*/2 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:          
+            app: keycloak-sync
+        spec:
+          initContainers:
+            {{- include "certificates.initContainer" . | nindent 12 }}
+          containers:
+            - name: keycloak-sync
+              image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}"
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: DB_HOST
+                  value: {{ template "postgresql.fullname" . }}
+                - name: DB_USER
+                  value: {{ .Values.global.db.common.username }}
+                - name: DB_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.db.common.passwordSecretName }}
+                      key: password
+                - name: KEYCLOAK_URL
+                  value: {{ include "renku.keycloakUrl" . | quote}}
+                - name: KEYCLOAK_REALM
+                  value: {{ include "renku.keycloak.realm" . | quote}}
+                - name: KEYCLOAK_CLIENT_ID
+                  value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+                - name: KEYCLOAK_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ include "renku.fullname" . }}
+                      key: dataServiceKeycloakClientSecret
+                - name: TOTAL_USER_SYNC
+                  value: "false"
+                {{- include "certificates.env.python" . | nindent 16 }}
+              volumeMounts:
+                {{- include "certificates.volumeMounts.system" . | nindent 16 }}
+          restartPolicy: Never
+          volumes:
+            {{- include "certificates.volumes" . | nindent 12 }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "renku.fullname" . }}-keycloak-sync-total
+  labels:
+    app: keycloak-sync
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ template "renku.chart" . }}
+spec:
+  schedule: "0 3 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:          
+            app: keycloak-sync
+        spec:
+          initContainers:
+            {{- include "certificates.initContainer" . | nindent 12 }}
+          containers:
+            - name: keycloak-sync
+              image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}"
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: DB_HOST
+                  value: {{ template "postgresql.fullname" . }}
+                - name: DB_USER
+                  value: {{ .Values.global.db.common.username }}
+                - name: DB_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.db.common.passwordSecretName }}
+                      key: password
+                - name: KEYCLOAK_URL
+                  value: {{ include "renku.keycloakUrl" . | quote}}
+                - name: KEYCLOAK_REALM
+                  value: {{ include "renku.keycloak.realm" . | quote}}
+                - name: KEYCLOAK_CLIENT_ID
+                  value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+                - name: KEYCLOAK_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ include "renku.fullname" . }}
+                      key: dataServiceKeycloakClientSecret
+                - name: TOTAL_USER_SYNC
+                  value: "true"
+                {{- include "certificates.env.python" . | nindent 16 }}
+              volumeMounts:
+                {{- include "certificates.volumeMounts.system" . | nindent 16 }}
+          restartPolicy: Never
+          volumes:
+            {{- include "certificates.volumes" . | nindent 12 }}

helm-chart/renku/templates/network-policies.yaml

@@ -69,6 +69,12 @@ spec:
           namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        - podSelector:
+            matchLabels:
+              app: keycloak-sync
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
       ports:
         - protocol: TCP
           port: 5432

helm-chart/renku/templates/search/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "renku.search.searchApi.name" -}}
+{{- "search-api" -}}
+{{- end -}}
+
+{{- define "renku.search.searchProvision.name" -}}
+{{- "search-provision" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "renku.search.searchApi.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- printf "%s-search-api" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-search-api" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "renku.search.searchProvision.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- printf "%s-search-provision" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-search-provision" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}

helm-chart/renku/templates/search/search-api-deployment.yaml

@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "renku.search.searchApi.fullname" . }}
+  labels:
+    app: {{ template "renku.search.searchApi.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.search.searchApi.replicas }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: {{ template "renku.search.searchApi.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "renku.search.searchApi.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      automountServiceAccountToken: {{ .Values.global.debug }}
+      containers:
+        - name: search-api
+          image: "{{ .Values.search.searchApi.image.repository }}:{{ .Values.search.searchApi.image.tag }}"
+          imagePullPolicy: {{ .Values.search.searchApi.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          env:
+            - name: RS_SOLR_URL
+              value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr"
+            - name: RS_SOLR_CORE
+              value: {{ first .Values.solr.coreNames }}
+            - name: RS_SOLR_USER
+              value: {{ .Values.solr.auth.adminUsername | quote }}
+            - name: RS_SOLR_PASS
+              value: {{ .Values.solr.auth.adminPassword | quote }}
+            - name: RS_SOLR_DEFAULT_COMMIT_WITHIN
+              value: "500ms"
+            - name: RS_SOLR_LOG_MESSAGE_BODIES
+              value: "false"
+            - name: JAVA_OPTS
+              value: "-Xmx{{ .Values.search.searchApi.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational"
+          ports:
+            - name: http-search-api
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http-search-api
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http-search-api
+          resources:
+            {{- toYaml .Values.search.searchApi.resources | nindent 12 }}
+          volumeMounts:
+            {{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }}
+      volumes:
+        {{- include "certificates.volumes" . | nindent 8 }}
+  {{- with .Values.nodeSelector }}
+nodeSelector:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.affinity }}
+affinity:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.tolerations }}
+tolerations:
+  {{ toYaml . | indent 8 }}
+  {{- end }}

helm-chart/renku/templates/search/search-api-service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "renku.search.searchApi.fullname" . }}
+  labels:
+    app: {{ template "renku.search.searchApi.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.search.searchApi.service.type }}
+  ports:
+    - port: {{ .Values.search.searchApi.service.port }}
+      targetPort: http-search-api
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ template "renku.search.searchApi.name" . }}
+    release: {{ .Release.Name }}