Skip to content

Commit

Permalink
feat: renku 2.0 search (#3478)
Browse files Browse the repository at this point in the history
* chore: Renku search helm charts (#3481)

* feat: add Solr to the subcharts.

* feat: add message queue Redis to data-services (#3508)

* chore: add gateway config for search endpoint

* chore: add redis values for users sync svc

* chore: add redis access to user sync job
  • Loading branch information
lorenzo-cavazzi authored Mar 27, 2024
1 parent c1f078c commit 85c8202
Show file tree
Hide file tree
Showing 15 changed files with 416 additions and 14 deletions.
18 changes: 18 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,24 @@
0.50.x
------

This release introduces new search functionality for Renku 2.0


Internal Changes
~~~~~~~~~~~~~~~~

**New Features**

- **Data services**: Initial support for project and user search for Renku 2.0 (alpha release)
- **Search services**: Initial support for project and user search for Renku 2.0 (alpha release)

Individual Components
~~~~~~~~~~~~~~~~~~~~~

- `renku-data-services 0.6.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.6.0>`_
- `renku-gateway 0.24.0 <https://github.com/SwissDataScienceCenter/renku-gateway/releases/0.24.0>`_
- `renku-search 0.0.37 <https://github.com/SwissDataScienceCenter/renku-search/releases/tag/v0.0.37>`_



0.49.0
Expand Down
4 changes: 4 additions & 0 deletions helm-chart/renku/requirements.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,7 @@ dependencies:
repository: "https://swissdatasciencecenter.github.io/helm-charts/"
version: "0.1.7"
condition: global.csi-rclone.install
- name: solr
repository: "oci://registry-1.docker.io/bitnamicharts"
version: "8.9.2"
condition: solr.enabled
4 changes: 4 additions & 0 deletions helm-chart/renku/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ Define subcharts full names
{{- printf "%s-%s" .Release.Name "keycloakx" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{- define "solr.fullname" -}}
{{- printf "%s-%s" .Release.Name "solr" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{- define "gitlab.fullname" -}}
{{- printf "%s-%s" .Release.Name "gitlab" | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
Expand Down
16 changes: 16 additions & 0 deletions helm-chart/renku/templates/data-service/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ spec:
labels:
app: renku-data-service
release: {{ .Release.Name }}
{{ .Values.global.redis.clientLabel | toYaml | nindent 8 }}
{{- with .Values.dataService.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -75,6 +76,21 @@ spec:
value: {{ .Values.global.gitlab.url | quote }}
- name: MAX_PINNED_PROJECTS
value: {{ .Values.dataService.maxPinnedProjects | quote }}
- name: REDIS_HOST
value: {{ .Values.global.redis.host | quote }}
- name: REDIS_PORT
value: {{ .Values.global.redis.port | quote }}
- name: REDIS_DATABASE
value: {{ .Values.global.redis.dbIndex.events| quote }}
- name: REDIS_IS_SENTINEL
value: {{ .Values.global.redis.sentinel.enabled | quote }}
- name: REDIS_MASTER_SET
value: {{ .Values.global.redis.sentinel.masterSet | quote }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.global.redis.existingSecret }}
key: {{ .Values.global.redis.existingSecretPasswordKey }}
{{- include "certificates.env.python" $ | nindent 12 }}
volumeMounts:
- name: server-options
Expand Down
2 changes: 2 additions & 0 deletions helm-chart/renku/templates/gateway/deployment-revproxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ spec:
value: {{ printf "http://%s-gateway-auth" .Release.Name }}
- name: REVPROXY_RENKU_SERVICES_DATA_SERVICE
value: {{ printf "http://%s-data-service" .Release.Name | quote }}
- name: REVPROXY_RENKU_SERVICES_SEARCH
value: {{ printf "http://%s-search-api" .Release.Name | quote }}
- name: REVPROXY_RENKU_SERVICES_KEYCLOAK
value: {{ include "renku.keycloakUrl" . | quote }}
- name: REVPROXY_PORT
Expand Down
1 change: 1 addition & 0 deletions helm-chart/renku/templates/ingress.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
{{- $graphEnabled := .Values.graph.enabled -}}
{{- $webhookServiceFullname := include "renku.graph.webhookService.fullname" . -}}
{{- $knowledgeGraphFullname := include "renku.graph.knowledgeGraph.fullname" . -}}
{{- $searchApiFullname := include "renku.search.searchApi.fullname" . -}}
{{- $renkuFullname := include "renku.fullname" . -}}
{{- if $graphEnabled }}
{{- $jenaFullname := include "renku-jena.fullname" . -}}
Expand Down
33 changes: 33 additions & 0 deletions helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ spec:
metadata:
labels:
app: keycloak-sync
{{ .Values.global.redis.clientLabel | toYaml | nindent 12 }}
spec:
initContainers:
{{- include "certificates.initContainer" . | nindent 12 }}
Expand Down Expand Up @@ -48,6 +49,21 @@ spec:
key: dataServiceKeycloakClientSecret
- name: TOTAL_USER_SYNC
value: "false"
- name: REDIS_HOST
value: {{ .Values.global.redis.host | quote }}
- name: REDIS_PORT
value: {{ .Values.global.redis.port | quote }}
- name: REDIS_DATABASE
value: {{ .Values.global.redis.dbIndex.events| quote }}
- name: REDIS_IS_SENTINEL
value: {{ .Values.global.redis.sentinel.enabled | quote }}
- name: REDIS_MASTER_SET
value: {{ .Values.global.redis.sentinel.masterSet | quote }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.global.redis.existingSecret }}
key: {{ .Values.global.redis.existingSecretPasswordKey }}
{{- include "certificates.env.python" . | nindent 16 }}
volumeMounts:
{{- include "certificates.volumeMounts.system" . | nindent 16 }}
Expand All @@ -64,6 +80,7 @@ metadata:
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
chart: {{ template "renku.chart" . }}
{{ .Values.global.redis.clientLabel | toYaml | nindent 4 }}
spec:
schedule: "0 3 * * *"
concurrencyPolicy: Forbid
Expand All @@ -75,6 +92,7 @@ spec:
metadata:
labels:
app: keycloak-sync
{{ .Values.global.redis.clientLabel | toYaml | nindent 12 }}
spec:
initContainers:
{{- include "certificates.initContainer" . | nindent 12 }}
Expand Down Expand Up @@ -105,6 +123,21 @@ spec:
key: dataServiceKeycloakClientSecret
- name: TOTAL_USER_SYNC
value: "true"
- name: REDIS_HOST
value: {{ .Values.global.redis.host | quote }}
- name: REDIS_PORT
value: {{ .Values.global.redis.port | quote }}
- name: REDIS_DATABASE
value: {{ .Values.global.redis.dbIndex.events| quote }}
- name: REDIS_IS_SENTINEL
value: {{ .Values.global.redis.sentinel.enabled | quote }}
- name: REDIS_MASTER_SET
value: {{ .Values.global.redis.sentinel.masterSet | quote }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.global.redis.existingSecret }}
key: {{ .Values.global.redis.existingSecretPasswordKey }}
{{- include "certificates.env.python" . | nindent 16 }}
volumeMounts:
{{- include "certificates.volumeMounts.system" . | nindent 16 }}
Expand Down
32 changes: 32 additions & 0 deletions helm-chart/renku/templates/search/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "renku.search.searchApi.name" -}}
{{- "search-api" -}}
{{- end -}}

{{- define "renku.search.searchProvision.name" -}}
{{- "search-provision" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "renku.search.searchApi.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- printf "%s-search-api" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-search-api" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}

{{- define "renku.search.searchProvision.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- printf "%s-search-provision" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-search-provision" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
80 changes: 80 additions & 0 deletions helm-chart/renku/templates/search/search-api-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "renku.search.searchApi.fullname" . }}
labels:
app: {{ template "renku.search.searchApi.name" . }}
chart: {{ template "renku.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
replicas: {{ .Values.search.searchApi.replicas }}
strategy:
type: Recreate
selector:
matchLabels:
app: {{ template "renku.search.searchApi.name" . }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ template "renku.search.searchApi.name" . }}
release: {{ .Release.Name }}
spec:
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
automountServiceAccountToken: {{ .Values.global.debug }}
containers:
- name: search-api
image: "{{ .Values.search.searchApi.image.repository }}:{{ .Values.search.searchApi.image.tag }}"
imagePullPolicy: {{ .Values.search.searchApi.image.pullPolicy }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
env:
- name: RS_SOLR_URL
value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr"
- name: RS_SOLR_CORE
value: {{ first .Values.solr.coreNames }}
- name: RS_SOLR_USER
value: {{ default "admin" .Values.solr.auth.adminUsername | quote }}
- name: RS_SOLR_PASS
valueFrom:
secretKeyRef:
name: {{ template "solr.fullname" . }}
key: solr-password
- name: RS_SOLR_DEFAULT_COMMIT_WITHIN
value: "500ms"
- name: RS_SOLR_LOG_MESSAGE_BODIES
value: "false"
- name: JAVA_OPTS
value: "-Xmx{{ .Values.search.searchApi.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational"
ports:
- name: http-search-api
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
path: /ping
port: http-search-api
readinessProbe:
httpGet:
path: /ping
port: http-search-api
resources:
{{- toYaml .Values.search.searchApi.resources | nindent 12 }}
volumeMounts:
{{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }}
volumes:
{{- include "certificates.volumes" . | nindent 8 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
23 changes: 23 additions & 0 deletions helm-chart/renku/templates/search/search-api-service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
apiVersion: v1
kind: Service
metadata:
name: {{ template "renku.search.searchApi.fullname" . }}
labels:
app: {{ template "renku.search.searchApi.name" . }}
chart: {{ template "renku.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8080'
spec:
type: {{ .Values.search.searchApi.service.type }}
ports:
- port: {{ .Values.search.searchApi.service.port }}
targetPort: http-search-api
protocol: TCP
name: http
selector:
app: {{ template "renku.search.searchApi.name" . }}
release: {{ .Release.Name }}
93 changes: 93 additions & 0 deletions helm-chart/renku/templates/search/search-provision-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "renku.search.searchProvision.fullname" . }}
labels:
app: {{ template "renku.search.searchProvision.name" . }}
chart: {{ template "renku.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
replicas: {{ .Values.search.searchProvision.replicas }}
strategy:
type: Recreate
selector:
matchLabels:
app: {{ template "renku.search.searchProvision.name" . }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ template "renku.search.searchProvision.name" . }}
release: {{ .Release.Name }}
# The label below enables to connect to redis
{{ .Values.global.redis.clientLabel | toYaml | nindent 8 }}
spec:
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
automountServiceAccountToken: {{ .Values.global.debug }}
containers:
- name: search-provision
image: "{{ .Values.search.searchProvision.image.repository }}:{{ .Values.search.searchProvision.image.tag }}"
imagePullPolicy: {{ .Values.search.searchProvision.image.pullPolicy }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
env:
- name: RS_REDIS_HOST
value: {{ .Values.global.redis.host | quote }}
- name: RS_REDIS_PORT
value: {{ .Values.global.redis.port | quote }}
- name: RS_REDIS_SENTINEL
value: {{ .Values.global.redis.sentinel.enabled | quote }}
- name: RS_REDIS_DB
value: {{ .Values.global.redis.dbIndex.events | quote }}
- name: RS_REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.global.redis.existingSecret }}
key: {{ .Values.global.redis.existingSecretPasswordKey }}
- name: RS_REDIS_MASTER_SET
value: {{ .Values.global.redis.sentinel.masterSet | quote }}
{{- range $k, $v := .Values.global.events.streams }}
- name: RS_REDIS_QUEUE_{{ $k }}
value: {{ $v }}
{{- end }}
- name: RS_SOLR_URL
value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr"
- name: RS_SOLR_CORE
value: {{ first .Values.solr.coreNames }}
- name: RS_SOLR_USER
value: {{ default "admin" .Values.solr.auth.adminUsername | quote }}
- name: RS_SOLR_PASS
valueFrom:
secretKeyRef:
name: {{ template "solr.fullname" . }}
key: solr-password
- name: RS_SOLR_DEFAULT_COMMIT_WITHIN
value: "500ms"
- name: RS_SOLR_LOG_MESSAGE_BODIES
value: "false"
- name: JAVA_OPTS
value: "-Xmx{{ .Values.search.searchProvision.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational"
ports:
- name: http-searchprov
containerPort: 8081
protocol: TCP
resources:
{{- toYaml .Values.search.searchProvision.resources | nindent 12 }}
volumeMounts:
{{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }}
volumes:
{{- include "certificates.volumes" . | nindent 8 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
Loading

0 comments on commit 85c8202

Please sign in to comment.