feat: Search and Discovery 2.0

CHANGELOG.rst

@@ -3,6 +3,24 @@
 0.50.x
 ------
 
+This release introduces new search functionality for Renku 2.0
+
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**New Features**
+
+- **Data services**: Initial support for project and user search for Renku 2.0 (alpha release)
+- **Search services**: Initial support for project and user search for Renku 2.0 (alpha release)
+
+Individual Components
+~~~~~~~~~~~~~~~~~~~~~
+
+- `renku-data-services 0.6.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.6.0>`_
+- `renku-gateway 0.24.0 <https://github.com/SwissDataScienceCenter/renku-gateway/releases/0.24.0>`_
+- `renku-search 0.0.37 <https://github.com/SwissDataScienceCenter/renku-search/releases/tag/v0.0.37>`_
+
 
 
 0.49.0

helm-chart/renku/requirements.yaml

@@ -32,3 +32,7 @@ dependencies:
     repository: "https://swissdatasciencecenter.github.io/helm-charts/"
     version: "0.1.7"
     condition: global.csi-rclone.install
+  - name: solr
+    repository: "oci://registry-1.docker.io/bitnamicharts"
+    version: "8.9.2"
+    condition: solr.enabled

helm-chart/renku/templates/_helpers.tpl

@@ -57,6 +57,10 @@ Define subcharts full names
 {{- printf "%s-%s" .Release.Name "keycloakx" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{- define "solr.fullname" -}}
+{{- printf "%s-%s" .Release.Name "solr" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "gitlab.fullname" -}}
 {{- printf "%s-%s" .Release.Name "gitlab" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

helm-chart/renku/templates/data-service/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       labels:
         app: renku-data-service
         release: {{ .Release.Name }}
+        {{ .Values.global.redis.clientLabel | toYaml | nindent 8 }}
       {{- with .Values.dataService.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
@@ -75,6 +76,21 @@ spec:
               value: {{ .Values.global.gitlab.url | quote }}
             - name: MAX_PINNED_PROJECTS
               value: {{ .Values.dataService.maxPinnedProjects | quote }}
+            - name: REDIS_HOST
+              value: {{ .Values.global.redis.host | quote }}
+            - name: REDIS_PORT
+              value: {{ .Values.global.redis.port | quote }}
+            - name: REDIS_DATABASE
+              value: {{ .Values.global.redis.dbIndex.events| quote }}
+            - name: REDIS_IS_SENTINEL
+              value: {{ .Values.global.redis.sentinel.enabled | quote }}
+            - name: REDIS_MASTER_SET
+              value: {{ .Values.global.redis.sentinel.masterSet | quote }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.redis.existingSecret }}
+                  key: {{ .Values.global.redis.existingSecretPasswordKey }}
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: server-options

helm-chart/renku/templates/gateway/deployment-revproxy.yaml

@@ -69,6 +69,8 @@ spec:
               value: {{ printf "http://%s-gateway-auth" .Release.Name }}
             - name: REVPROXY_RENKU_SERVICES_DATA_SERVICE
               value: {{ printf "http://%s-data-service" .Release.Name | quote }}
+            - name: REVPROXY_RENKU_SERVICES_SEARCH
+              value: {{ printf "http://%s-search-api" .Release.Name | quote }}
             - name: REVPROXY_RENKU_SERVICES_KEYCLOAK
               value: {{ include "renku.keycloakUrl" . | quote }}
             - name: REVPROXY_PORT

helm-chart/renku/templates/ingress.yaml

@@ -12,6 +12,7 @@
 {{- $graphEnabled := .Values.graph.enabled -}}
 {{- $webhookServiceFullname := include "renku.graph.webhookService.fullname" . -}}
 {{- $knowledgeGraphFullname := include "renku.graph.knowledgeGraph.fullname" . -}}
+{{- $searchApiFullname := include "renku.search.searchApi.fullname" . -}}
 {{- $renkuFullname := include "renku.fullname" . -}}
 {{- if $graphEnabled }}
 {{- $jenaFullname := include "renku-jena.fullname" . -}}

helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml

@@ -18,6 +18,7 @@ spec:
         metadata:
           labels:          
             app: keycloak-sync
+            {{ .Values.global.redis.clientLabel | toYaml | nindent 12 }}
         spec:
           initContainers:
             {{- include "certificates.initContainer" . | nindent 12 }}
@@ -48,6 +49,21 @@ spec:
                       key: dataServiceKeycloakClientSecret
                 - name: TOTAL_USER_SYNC
                   value: "false"
+                - name: REDIS_HOST
+                  value: {{ .Values.global.redis.host | quote }}
+                - name: REDIS_PORT
+                  value: {{ .Values.global.redis.port | quote }}
+                - name: REDIS_DATABASE
+                  value: {{ .Values.global.redis.dbIndex.events| quote }}
+                - name: REDIS_IS_SENTINEL
+                  value: {{ .Values.global.redis.sentinel.enabled | quote }}
+                - name: REDIS_MASTER_SET
+                  value: {{ .Values.global.redis.sentinel.masterSet | quote }}
+                - name: REDIS_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.redis.existingSecret }}
+                      key: {{ .Values.global.redis.existingSecretPasswordKey }}
                 {{- include "certificates.env.python" . | nindent 16 }}
               volumeMounts:
                 {{- include "certificates.volumeMounts.system" . | nindent 16 }}
@@ -64,6 +80,7 @@ metadata:
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
     chart: {{ template "renku.chart" . }}
+    {{ .Values.global.redis.clientLabel | toYaml | nindent 4 }}
 spec:
   schedule: "0 3 * * *"
   concurrencyPolicy: Forbid
@@ -75,6 +92,7 @@ spec:
         metadata:
           labels:          
             app: keycloak-sync
+            {{ .Values.global.redis.clientLabel | toYaml | nindent 12 }}
         spec:
           initContainers:
             {{- include "certificates.initContainer" . | nindent 12 }}
@@ -105,6 +123,21 @@ spec:
                       key: dataServiceKeycloakClientSecret
                 - name: TOTAL_USER_SYNC
                   value: "true"
+                - name: REDIS_HOST
+                  value: {{ .Values.global.redis.host | quote }}
+                - name: REDIS_PORT
+                  value: {{ .Values.global.redis.port | quote }}
+                - name: REDIS_DATABASE
+                  value: {{ .Values.global.redis.dbIndex.events| quote }}
+                - name: REDIS_IS_SENTINEL
+                  value: {{ .Values.global.redis.sentinel.enabled | quote }}
+                - name: REDIS_MASTER_SET
+                  value: {{ .Values.global.redis.sentinel.masterSet | quote }}
+                - name: REDIS_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.redis.existingSecret }}
+                      key: {{ .Values.global.redis.existingSecretPasswordKey }}
                 {{- include "certificates.env.python" . | nindent 16 }}
               volumeMounts:
                 {{- include "certificates.volumeMounts.system" . | nindent 16 }}

helm-chart/renku/templates/search/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "renku.search.searchApi.name" -}}
+{{- "search-api" -}}
+{{- end -}}
+
+{{- define "renku.search.searchProvision.name" -}}
+{{- "search-provision" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "renku.search.searchApi.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- printf "%s-search-api" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-search-api" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "renku.search.searchProvision.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- printf "%s-search-provision" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-search-provision" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}

helm-chart/renku/templates/search/search-api-deployment.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "renku.search.searchApi.fullname" . }}
+  labels:
+    app: {{ template "renku.search.searchApi.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.search.searchApi.replicas }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: {{ template "renku.search.searchApi.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "renku.search.searchApi.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      automountServiceAccountToken: {{ .Values.global.debug }}
+      containers:
+        - name: search-api
+          image: "{{ .Values.search.searchApi.image.repository }}:{{ .Values.search.searchApi.image.tag }}"
+          imagePullPolicy: {{ .Values.search.searchApi.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          env:
+            - name: RS_SOLR_URL
+              value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr"
+            - name: RS_SOLR_CORE
+              value: {{ first .Values.solr.coreNames }}
+            - name: RS_SOLR_USER
+              value: {{ default "admin" .Values.solr.auth.adminUsername | quote }}
+            - name: RS_SOLR_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "solr.fullname" . }}
+                  key: solr-password
+            - name: RS_SOLR_DEFAULT_COMMIT_WITHIN
+              value: "500ms"
+            - name: RS_SOLR_LOG_MESSAGE_BODIES
+              value: "false"
+            - name: JAVA_OPTS
+              value: "-Xmx{{ .Values.search.searchApi.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational"
+          ports:
+            - name: http-search-api
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http-search-api
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http-search-api
+          resources:
+            {{- toYaml .Values.search.searchApi.resources | nindent 12 }}
+          volumeMounts:
+            {{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }}
+      volumes:
+        {{- include "certificates.volumes" . | nindent 8 }}
+  {{- with .Values.nodeSelector }}
+nodeSelector:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.affinity }}
+affinity:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.tolerations }}
+tolerations:
+  {{ toYaml . | indent 8 }}
+  {{- end }}

helm-chart/renku/templates/search/search-api-service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "renku.search.searchApi.fullname" . }}
+  labels:
+    app: {{ template "renku.search.searchApi.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/path: '/metrics'
+    prometheus.io/port: '8080'
+spec:
+  type: {{ .Values.search.searchApi.service.type }}
+  ports:
+    - port: {{ .Values.search.searchApi.service.port }}
+      targetPort: http-search-api
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ template "renku.search.searchApi.name" . }}
+    release: {{ .Release.Name }}

helm-chart/renku/templates/search/search-provision-deployment.yaml

@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "renku.search.searchProvision.fullname" . }}
+  labels:
+    app: {{ template "renku.search.searchProvision.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.search.searchProvision.replicas }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: {{ template "renku.search.searchProvision.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "renku.search.searchProvision.name" . }}
+        release: {{ .Release.Name }}
+        # The label below enables to connect to redis
+        {{ .Values.global.redis.clientLabel | toYaml | nindent 8 }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      automountServiceAccountToken: {{ .Values.global.debug }}
+      containers:
+        - name: search-provision
+          image: "{{ .Values.search.searchProvision.image.repository }}:{{ .Values.search.searchProvision.image.tag }}"
+          imagePullPolicy: {{ .Values.search.searchProvision.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          env:
+            - name: RS_REDIS_HOST
+              value: {{ .Values.global.redis.host | quote }}
+            - name: RS_REDIS_PORT
+              value: {{ .Values.global.redis.port | quote }}
+            - name: RS_REDIS_SENTINEL
+              value: {{ .Values.global.redis.sentinel.enabled | quote }}
+            - name: RS_REDIS_DB
+              value: {{ .Values.global.redis.dbIndex.events | quote }}
+            - name: RS_REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.redis.existingSecret }}
+                  key: {{ .Values.global.redis.existingSecretPasswordKey }}
+            - name: RS_REDIS_MASTER_SET
+              value: {{ .Values.global.redis.sentinel.masterSet | quote }}
+            {{- range $k, $v := .Values.global.events.streams }}
+            - name: RS_REDIS_QUEUE_{{ $k }}
+              value: {{ $v }}
+            {{- end }}
+            - name: RS_SOLR_URL
+              value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr"
+            - name: RS_SOLR_CORE
+              value: {{ first .Values.solr.coreNames }}
+            - name: RS_SOLR_USER
+              value: {{ default "admin" .Values.solr.auth.adminUsername | quote }}
+            - name: RS_SOLR_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "solr.fullname" . }}
+                  key: solr-password
+            - name: RS_SOLR_DEFAULT_COMMIT_WITHIN
+              value: "500ms"
+            - name: RS_SOLR_LOG_MESSAGE_BODIES
+              value: "false"
+            - name: JAVA_OPTS
+              value: "-Xmx{{ .Values.search.searchProvision.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational"
+          ports:
+            - name: http-searchprov
+              containerPort: 8081
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.search.searchProvision.resources | nindent 12 }}
+          volumeMounts:
+            {{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }}
+      volumes:
+        {{- include "certificates.volumes" . | nindent 8 }}
+  {{- with .Values.nodeSelector }}
+nodeSelector:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.affinity }}
+affinity:
+  {{ toYaml . | indent 8 }}
+  {{- end }}
+  {{- with .Values.tolerations }}
+tolerations:
+  {{ toYaml . | indent 8 }}
+  {{- end }}