From 79e08bb074099c6a431276054f88173748740e66 Mon Sep 17 00:00:00 2001 From: Alessandro Degano <40891147+aledegano@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:03:38 +0100 Subject: [PATCH 01/61] chore: remove Github action that opens PRs in `terraform-renku` to update Renkulab. (#3496) Going forward @yat will rely on Renovate to create a PR in `terraform-renku` that updates the version of Renkulab. --- .github/workflows/publish-helm-chart.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 873d5b85a6..6a8aa913b7 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -40,16 +40,3 @@ jobs: if: always() outputs: chart-version: ${{ steps.vars.outputs.tag }} - rollout-renku-deployments: - runs-on: ubuntu-20.04 - needs: - - "publish-chart" - steps: - - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - - name: Rollout renku version - uses: SwissDataScienceCenter/renku-actions/rollout-renku-version@v1.10.0 - env: - CHART_VERSION: ${{ needs.publish-chart.outputs.chart-version }} - GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} From f194e6e84b0ffdd5090baa9e6a1db718c0d88dbb Mon Sep 17 00:00:00 2001 From: Lorenzo Cavazzi <43481553+lorenzo-cavazzi@users.noreply.github.com> Date: Fri, 16 Feb 2024 15:11:43 +0100 Subject: [PATCH 02/61] fix: change the UI privacy configmap behavior (#3494) --- CHANGELOG.rst | 6 +++++ docs/how-to-guides/admin/privacycookie.rst | 27 ++++++++++--------- .../templates/ui/ui-client-configmap.yaml | 8 +++--- .../ui/ui-client-deployment-template.yaml | 14 +++++++--- helm-chart/renku/values.yaml | 11 ++++---- helm-chart/values.yaml.changelog.md | 11 ++++++++ 6 files changed, 52 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2237a00aa2..3c37c9b94e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,11 @@ .. _changelog: +0.48.1 +------ + +Renku ``0.48.1`` only changes how the Terms of Use and Privacy Policy sections +can be customized by administrators. + 0.48.0 ------ diff --git a/docs/how-to-guides/admin/privacycookie.rst b/docs/how-to-guides/admin/privacycookie.rst index 6c9c904090..1e2472f7ed 100644 --- a/docs/how-to-guides/admin/privacycookie.rst +++ b/docs/how-to-guides/admin/privacycookie.rst @@ -9,19 +9,20 @@ Privacy page and Terms of Use The UI can be configured to show a `Privacy Policy` and `Terms of Use`. These are displayed under the `Help` section of the UI. -For each of these, the content is read from a ``ConfigMap``. You need to configure -the values in ``ui.client.privacy.page`` to enable the feature and set the reference -ConfigMap name and key. If ``ui.client.privacy.page.enabled`` is ``true``, then the privacy -policy and terms of use will be shown in the UI, with content taken from the ConfigMap -specified by ``ui.client.privacy.page.configMapName`` at the key -``ui.client.privacy.page.configMapPolicyKey`` for the privacy policy and -``ui.client.privacy.page.configMapTermsKey`` for the terms of use. - -.. note:: - - If you don't set the ConfigMap name and key, - `a sample `_ - will be used instead. You can start from it as a template to create your customized ConfigMap. +For each of these, the content is read from the ``privacy-and-terms`` ConfigMap. +You need to configure the values in ``ui.client.privacy.page`` to enable the feature. +If ``ui.client.privacy.page.enabled`` is ``true``, then the privacy +policy and terms of use will be shown in the UI. + +Mind that you need to customize the content by either changing the ``privacy_statement`` +and the ``terms`` keys in the ``privacy-and-terms`` ConfigMap, or assign a valid Markdown +text to ``ui.client.privacy.page.privacyPolicyContent`` and +``ui.client.privacy.page.termsContent``. + +If you don't customize the text, +`a sample `_ +will be used instead. You can start from it as a template to create your own +content. The `Markdown syntax `_ is fully supported for the privacy page content. diff --git a/helm-chart/renku/templates/ui/ui-client-configmap.yaml b/helm-chart/renku/templates/ui/ui-client-configmap.yaml index 75c0e3cfcd..89b46d1f0a 100644 --- a/helm-chart/renku/templates/ui/ui-client-configmap.yaml +++ b/helm-chart/renku/templates/ui/ui-client-configmap.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "renku.fullname" . }}-privacy-sample + name: {{ template "renku.fullname" . }}-privacy-and-terms labels: app: ui chart: {{ template "renku.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - privacy_statement: | + sample_privacy_statement: | # Privacy statement The content of this page is only a template. ## Information @@ -25,7 +25,7 @@ data: ## Apply the changes If you edit the ConfigMap content and you don't upgrade the deployment from helm, keep in mind that the ui pod needs to be manually re-deployed in order to apply the changes to the privacy page. - terms: | + sample_terms: | # Terms of Use The content of this page is only a template. ## Information @@ -41,3 +41,5 @@ data: ## Apply the changes If you edit the ConfigMap content and you don't upgrade the deployment from helm, keep in mind that the ui pod needs to be manually re-deployed in order to apply the changes to the terms page. + privacy_statement: {{ .Values.ui.client.privacy.page.privacyPolicyContent | default ("") | quote }} + terms: {{ .Values.ui.client.privacy.page.termsContent | default ("") | quote }} diff --git a/helm-chart/renku/templates/ui/ui-client-deployment-template.yaml b/helm-chart/renku/templates/ui/ui-client-deployment-template.yaml index ae58ef7a41..334660a7da 100644 --- a/helm-chart/renku/templates/ui/ui-client-deployment-template.yaml +++ b/helm-chart/renku/templates/ui/ui-client-deployment-template.yaml @@ -29,11 +29,19 @@ spec: volumes: - name: privacy configMap: - name: {{ .Values.ui.client.privacy.page.configMapName | default (printf "%s-privacy-sample" (include "renku.fullname" .)) | quote }} + name: {{ printf "%s-privacy-and-terms" (include "renku.fullname" .) | quote }} items: - - key: {{ .Values.ui.client.privacy.page.configMapPolicyKey | default (printf "privacy_statement") | quote }} + {{- if .Values.ui.client.privacy.page.privacyPolicyContent }} + - key: "privacy_statement" + {{- else }} + - key: "sample_privacy_statement" + {{- end }} path: statement.md - - key: {{ .Values.ui.client.privacy.page.configMapTermsKey | default (printf "terms") | quote }} + {{- if .Values.ui.client.privacy.page.termsContent }} + - key: "terms" + {{- else }} + - key: "sample_terms" + {{- end }} path: terms.md {{- end }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index f5f5e750b1..b0ae4d8269 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -692,15 +692,14 @@ ui: dsn: "" environment: "" sampleRate: 0 # number between 0 and 1. (e.g., to send 20% of transactions, set 0.2.) - # If you want to enable the privacy page, please create also a configMap and set its name in the - # privacy.page.configMapName value. As a reference, you can use the sample configMap generated when - # enabling the feature. privacy: + # If you want to enable the Privacy Policy and Terms of Service pages, you shoud add your custom content + # in privacyPolicyContent and termsContent. Markdown is supported. + # As a reference, you can use the "ui-client-configmap" configMap content in the "sample" sections. page: enabled: false - #configMapName: privacy-page - #configMapPolicyKey: privacy_statement - #configMapTermsKey: terms + #privacyPolicyContent: "" + #termsContent: "" banner: enabled: false content: | diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index c4b2e1b961..41fdf5ee1d 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,17 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.48.1 + +The handling of privacy policy and terms of service content has been fine tuned. + +* DELETE `ui.client.privacy.page.configMapName` has been removed. +* DELETE `ui.client.privacy.page.configMapPolicyKey` has been removed. +* DELETE `ui.client.privacy.page.configMapTermsKey` has been removed. +* NEW ``ui.client.privacy.page.privacyPolicyContent`` to customize the content of the Privacy Policy page (supports Markdown). +* NEW ``ui.client.privacy.page.termsContent`` to customize the content of the Terms of Use page (supports Markdown). + + ## Upgrading to Renku 0.48.0 The handling of privacy policy and terms of service content has been slightly changed to make From e5e5803575f0980ef3e24f0b4012d34301ce78e2 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 19 Feb 2024 08:59:57 +0100 Subject: [PATCH 03/61] fake From 73add4f3786e640cf59ba5efd9072d129328b73e Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Mon, 19 Feb 2024 10:50:22 +0100 Subject: [PATCH 04/61] chore: Renku search helm charts (#3481) * chore(postgres): Upgrade Postgres to version 16.1.0 * feat: helm templates for renku-search services * feat: add Solr to the subcharts. * chore: search-api exposed to the world via /search * feat: default renku-search Solr core defined * chore: hardcoded Solr admin credentials * chore: search services connected to Solr * chore: Provisioner connected to Redis --- CHANGELOG.rst | 56 ++++++++ docs/renku-python | 2 +- helm-chart/renku/requirements.yaml | 9 +- helm-chart/renku/templates/_helpers.tpl | 4 + helm-chart/renku/templates/ingress.yaml | 8 ++ .../renku/templates/search/_helpers.tpl | 32 +++++ .../search/search-api-deployment.yaml | 77 +++++++++++ .../templates/search/search-api-service.yaml | 19 +++ .../search/search-provision-deployment.yaml | 84 ++++++++++++ .../renku/templates/setup-job-gitlab.yaml | 6 +- .../templates/setup-job-keycloak-db.yaml | 6 +- .../renku/templates/setup-job-renku-dbs.yaml | 6 +- helm-chart/renku/values.yaml | 120 +++++++++++++----- .../version_upgrades/psql_dump.yaml | 4 +- .../version_upgrades/psql_load.yaml | 4 +- helm-chart/values.yaml.changelog.md | 41 ++++++ 16 files changed, 431 insertions(+), 47 deletions(-) create mode 100644 helm-chart/renku/templates/search/_helpers.tpl create mode 100644 helm-chart/renku/templates/search/search-api-deployment.yaml create mode 100644 helm-chart/renku/templates/search/search-api-service.yaml create mode 100644 helm-chart/renku/templates/search/search-provision-deployment.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3c37c9b94e..927d55b099 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,4 +1,60 @@ .. _changelog: +0.xx.0 +------ + +Renku ``0.xx.0`` is a release that exclusively changes internal components and has no effect on the end-user. +Upgrades the PostgreSQL Helm chart version that is shipped with the Renku Helm chart. + +NOTE for administrators: make sure to check `helm-chart/values.yaml.changelog.md` for instructions on how to +upgrade to this version of Renku. +**Improvements** + +- **Infrastructure**: (BREAKING CHANGE) Upgrade the version of PostgreSQL to 16.1.0. This requires modifying your current values file to work with the new PostgreSQL Helm chart, see (`the helm chart values changelog `_) for instructions. + +0.48.0 +------ + +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**🌟 New Features** + +- 📜 **UI**: Show terms of use and privacy policy in the help section + (`#2954 `_). + +**✨ Improvements** + +- 🖌 **UI**: Improve appearance of templates on new project page + (`#2999 `_). +- 🛑 **UI**: Unify appearance of project settings alerts + (`#3001 `_). + +**🐞 Bug Fixes** + +- **UI**: Restore logged in/out notifications + (`#3014 `_). +- **UI**: Hide button to add storage on deployments not supporting external storages + (`#3001 `_). + +Internal Changes +~~~~~~~~~~~~~~~~ + +**Improvements** + +- **UI**: Add initial alpha implementation of Renku 1.0 projects + (`#2875 `_). + +Individual components +~~~~~~~~~~~~~~~~~~~~~~ + +- `renku-ui 3.20.0 `_ + + +0.49.0 +------ + +TODO: Search & Discovery 1.0 + 0.48.1 ------ diff --git a/docs/renku-python b/docs/renku-python index af89ce6ac1..a0a29bb8f7 160000 --- a/docs/renku-python +++ b/docs/renku-python @@ -1 +1 @@ -Subproject commit af89ce6ac12e00544158bacba87df82fdb03585f +Subproject commit a0a29bb8f7f4ff637ad3ffce19850f1eff06a2af diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 6b13e25c57..67baafd94a 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -4,8 +4,8 @@ dependencies: version: 0.8.0 condition: gitlab.enabled - name: postgresql - version: 9.1.1 - repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami" + version: 14.0.1 + repository: "https://charts.bitnami.com/bitnami" condition: postgresql.enabled - name: keycloakx version: 2.1.0 @@ -32,4 +32,7 @@ dependencies: repository: "https://swissdatasciencecenter.github.io/helm-charts/" version: "0.1.6" condition: global.csi-rclone.install - +- name: solr + repository: "oci://registry-1.docker.io/bitnamicharts" + version: "8.7.1" + condition: solr.enabled diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl index 7930555279..fb71c7d9ba 100644 --- a/helm-chart/renku/templates/_helpers.tpl +++ b/helm-chart/renku/templates/_helpers.tpl @@ -57,6 +57,10 @@ Define subcharts full names {{- printf "%s-%s" .Release.Name "keycloakx" | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- define "solr.fullname" -}} +{{- printf "%s-%s" .Release.Name "solr" | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "gitlab.fullname" -}} {{- printf "%s-%s" .Release.Name "gitlab" | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/helm-chart/renku/templates/ingress.yaml b/helm-chart/renku/templates/ingress.yaml index 331fe98217..d82240c1e7 100644 --- a/helm-chart/renku/templates/ingress.yaml +++ b/helm-chart/renku/templates/ingress.yaml @@ -12,6 +12,7 @@ {{- $graphEnabled := .Values.graph.enabled -}} {{- $webhookServiceFullname := include "renku.graph.webhookService.fullname" . -}} {{- $knowledgeGraphFullname := include "renku.graph.knowledgeGraph.fullname" . -}} +{{- $searchApiFullname := include "renku.search.searchApi.fullname" . -}} {{- $renkuFullname := include "renku.fullname" . -}} {{- if $graphEnabled }} {{- $jenaFullname := include "renku-jena.fullname" . -}} @@ -126,6 +127,13 @@ spec: port: number: 80 {{- end }} + - path: /search + pathType: Prefix + backend: + service: + name: {{ $searchApiFullname }} + port: + number: 80 {{- if $swaggerEnabled }} - path: /swagger pathType: Prefix diff --git a/helm-chart/renku/templates/search/_helpers.tpl b/helm-chart/renku/templates/search/_helpers.tpl new file mode 100644 index 0000000000..b37902d471 --- /dev/null +++ b/helm-chart/renku/templates/search/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "renku.search.searchApi.name" -}} +{{- "search-api" -}} +{{- end -}} + +{{- define "renku.search.searchProvision.name" -}} +{{- "search-provision" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "renku.search.searchApi.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- printf "%s-search-api" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-search-api" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "renku.search.searchProvision.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- printf "%s-search-provision" .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-search-provision" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} diff --git a/helm-chart/renku/templates/search/search-api-deployment.yaml b/helm-chart/renku/templates/search/search-api-deployment.yaml new file mode 100644 index 0000000000..0a7b571410 --- /dev/null +++ b/helm-chart/renku/templates/search/search-api-deployment.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "renku.search.searchApi.fullname" . }} + labels: + app: {{ template "renku.search.searchApi.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.search.searchApi.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + app: {{ template "renku.search.searchApi.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "renku.search.searchApi.name" . }} + release: {{ .Release.Name }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + automountServiceAccountToken: {{ .Values.global.debug }} + containers: + - name: search-api + image: "{{ .Values.search.searchApi.image.repository }}:{{ .Values.search.searchApi.image.tag }}" + imagePullPolicy: {{ .Values.search.searchApi.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + - name: RS_SOLR_URL + value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr" + - name: RS_SOLR_CORE + value: {{ first .Values.solr.coreNames }} + - name: RS_SOLR_USER + value: {{ .Values.solr.auth.adminUsername | quote }} + - name: RS_SOLR_PASS + value: {{ .Values.solr.auth.adminPassword | quote }} + - name: RS_SOLR_DEFAULT_COMMIT_WITHIN + value: "500ms" + - name: RS_SOLR_LOG_MESSAGE_BODIES + value: "false" + - name: JAVA_OPTS + value: "-Xmx{{ .Values.search.searchApi.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational" + ports: + - name: http-search-api + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /ping + port: http-search-api + readinessProbe: + httpGet: + path: /ping + port: http-search-api + resources: + {{- toYaml .Values.search.searchApi.resources | nindent 12 }} + volumeMounts: + {{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }} + volumes: + {{- include "certificates.volumes" . | nindent 8 }} + {{- with .Values.nodeSelector }} +nodeSelector: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} +affinity: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} +tolerations: + {{ toYaml . | indent 8 }} + {{- end }} diff --git a/helm-chart/renku/templates/search/search-api-service.yaml b/helm-chart/renku/templates/search/search-api-service.yaml new file mode 100644 index 0000000000..25c832e3c2 --- /dev/null +++ b/helm-chart/renku/templates/search/search-api-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "renku.search.searchApi.fullname" . }} + labels: + app: {{ template "renku.search.searchApi.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: {{ .Values.search.searchApi.service.type }} + ports: + - port: {{ .Values.search.searchApi.service.port }} + targetPort: http-search-api + protocol: TCP + name: http + selector: + app: {{ template "renku.search.searchApi.name" . }} + release: {{ .Release.Name }} diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml new file mode 100644 index 0000000000..a227df315d --- /dev/null +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "renku.search.searchProvision.fullname" . }} + labels: + app: {{ template "renku.search.searchProvision.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.search.searchProvision.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + app: {{ template "renku.search.searchProvision.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "renku.search.searchProvision.name" . }} + release: {{ .Release.Name }} + # The label below enables to connect to redis + {{ .Values.global.redis.clientLabel | toYaml | nindent 8 }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + automountServiceAccountToken: {{ .Values.global.debug }} + containers: + - name: search-provision + image: "{{ .Values.search.searchProvision.image.repository }}:{{ .Values.search.searchProvision.image.tag }}" + imagePullPolicy: {{ .Values.search.searchProvision.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + - name: RS_REDIS_HOST + value: {{ .Values.global.redis.host | quote }} + - name: RS_REDIS_PORT + value: {{ .Values.global.redis.port | quote }} + - name: RS_REDIS_SENTINEL + value: {{ .Values.global.redis.sentinel.enabled | quote }} + - name: RS_REDIS_DB + value: {{ .Values.global.redis.dbIndex.events | quote }} + - name: RS_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: {{ .Values.global.redis.existingSecretPasswordKey }} + - name: RS_REDIS_MASTER_SET + value: {{ .Values.global.redis.sentinel.masterSet | quote }} + - name: RS_REDIS_QUEUE_NAME + value: {{ first .Values.global.events.streams }} + - name: RS_SOLR_URL + value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr" + - name: RS_SOLR_CORE + value: {{ first .Values.solr.coreNames }} + - name: RS_SOLR_USER + value: {{ .Values.solr.auth.adminUsername | quote }} + - name: RS_SOLR_PASS + value: {{ .Values.solr.auth.adminPassword | quote }} + - name: RS_SOLR_DEFAULT_COMMIT_WITHIN + value: "500ms" + - name: RS_SOLR_LOG_MESSAGE_BODIES + value: "false" + - name: JAVA_OPTS + value: "-Xmx{{ .Values.search.searchProvision.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational" + resources: + {{- toYaml .Values.search.searchProvision.resources | nindent 12 }} + volumeMounts: + {{- include "certificates.volumeMounts.javaCertsGeneral" . | nindent 12 }} + volumes: + {{- include "certificates.volumes" . | nindent 8 }} + {{- with .Values.nodeSelector }} +nodeSelector: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} +affinity: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} +tolerations: + {{ toYaml . | indent 8 }} + {{- end }} diff --git a/helm-chart/renku/templates/setup-job-gitlab.yaml b/helm-chart/renku/templates/setup-job-gitlab.yaml index f01a4747d5..b6fc4cc58c 100644 --- a/helm-chart/renku/templates/setup-job-gitlab.yaml +++ b/helm-chart/renku/templates/setup-job-gitlab.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: GITLAB_ENABLED value: {{ .Values.gitlab.enabled | quote }} diff --git a/helm-chart/renku/templates/setup-job-keycloak-db.yaml b/helm-chart/renku/templates/setup-job-keycloak-db.yaml index 4d3229b2f0..68736d5441 100644 --- a/helm-chart/renku/templates/setup-job-keycloak-db.yaml +++ b/helm-chart/renku/templates/setup-job-keycloak-db.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: KEYCLOAK_DB_USERNAME value: {{ .Values.global.keycloak.postgresUser | quote }} diff --git a/helm-chart/renku/templates/setup-job-renku-dbs.yaml b/helm-chart/renku/templates/setup-job-renku-dbs.yaml index 50a49b7a7a..45dda94363 100644 --- a/helm-chart/renku/templates/setup-job-renku-dbs.yaml +++ b/helm-chart/renku/templates/setup-job-renku-dbs.yaml @@ -34,14 +34,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -50,7 +50,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: EVENTLOG_DB_USERNAME value: {{ .Values.global.graph.dbEventLog.postgresUser | quote }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index b0ae4d8269..a3a36a3ce4 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -79,6 +79,10 @@ global: ## for a new project based on its own version. ## NOTE should only be set for CI deployments and development purposes. cli_version: + ## Events related values definitions + events: + streams: + - project.created ## Note that the graph will not turned on by default until renku 0.4.0 graph: dbEventLog: @@ -129,6 +133,10 @@ global: ## postgresql-password key containing the password for the posgres user. # existingSecret: + # Globally available solr (TBD) + solr: + port: 8983 + # Connection details for a globally used redis instance for the # entire platform. For specifying an actual instance as part of # this chart, check out the non-global "redis" section. @@ -138,6 +146,7 @@ global: gateway: "0" coreService: "1" uiServer: "2" + events: "3" # Note: these two entries MUST match the ones in the top level redis section. existingSecret: redis-secret existingSecretPasswordKey: redis-password @@ -334,36 +343,43 @@ keycloakx: # For production deployments check out # https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values-production.yaml postgresql: - # If an external Postgres database is defined in global.externalServices.postgresql, - # postgresql.enabled should be false, and global.externalServices.postgresql.enabled should be true. - # By default, Renku-bundled Postgres is enabled. - enabled: true - ## We use the defaults here. - postgresqlDatabase: postgres - postgresqlUsername: postgres - ## The admin password should be set explicitly, otherwise a random string will be created. - ## Alternatively an existing secret can be provided. Note that postgres - ## DOES NOT tolerate a change of the admin password when upgrading. - # postgresqlPassword: + ## We use the defaults here. Note that these basic configs could also be set as + ## global values such that sub-charts can access them too. + + auth: + username: postgres + database: postgres + + ## The admin password should be set explicitly, otherwise a random string will be + ## created. Alternatively an existing secret can be provided. Note that postgres + ## DOES NOT tolerate a change of the admin password when upgrading. + # postgresqlPassword: + + ## Use an existing secret instead of creating a new one. It must have a + ## postgresql-password key containing the password for the postgres user. + # existingSecret: + + # Consider replication. These are the defaults for the basic settings. + # replicationUsername: repl_user + # replicationPassword: repl_password # generate a random password `openssl rand -hex 32` + + # image: + # repository: bitnami/postgresql + # tag: + + primary: + persistence: + ## We use the defaults here, but they will probably be modified for most deployments. + enabled: true + size: 8Gi + ## Provide an existing PersistentVolumeClaim to be reused. + # existingClaim: + + # Consider replication. These are the defaults for the basic settings. + readReplicas: + enabled: false + replicaCount: 1 - ## Use an existing secret instead of creating a new one. It must have a - ## postgresql-password key containing the password for the posgres user. - # existingSecret: - image: - repository: bitnami/postgresql - tag: 12.8.0 - persistence: - ## We use the defaults here, but they will probably be modified for most deployments. - enabled: true - size: 8Gi - ## Provide an existing PersistentVolumeClaim to be reused. - # existingClaim: - # Consider replication. These are the defaults for the basic settings. - replication: - enabled: false - user: repl_user - password: repl_password # generate a random password `openssl rand -hex 32` - slaveReplicas: 1 redis: # If set to true, a HA redis will be included in the Renku release. install: true @@ -465,6 +481,21 @@ redis: # default - making explicit sysctl: enabled: false +solr: + enabled: false + cloudEnabled: false + auth: + adminUsername: renku + adminPassword: renku + coreNames: + - renku-search + networkPolicy: + enabled: false # Awaiting instructions from: https://github.com/bitnami/charts/issues/23424 + persistence: + enabled: true + size: 8Gi + zookeeper: + enabled: false ## Gitlab configuration gitlab: ## Spawn a gitlab instance @@ -854,7 +885,8 @@ dlf-chart: enabled: false dataset-operator-chart: enabled: true -csi-rclone: {} +csi-rclone: + {} # This section is only relevant if you are installing csi-rclone as part of Renku ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster. # storageClassName: csi-rclone @@ -1296,6 +1328,34 @@ jena: additionalInitContainerScript: "find /fuseki -name tdb.lock -type f -delete && find /fuseki/configuration -name *.ttl -type f -empty -delete && echo clean-up done" test: enabled: false +## Configuration for renku-search services +search: + searchApi: + replicas: 1 + image: + repository: renku/search-api + tag: "0.0.11" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 80 + resources: + requests: + memory: 256Mi + jvmXmx: 256M + searchProvision: + replicas: 1 + image: + repository: renku/search-provision + tag: "0.0.11" + pullPolicy: IfNotPresent + service: + type: ClusterIP + resources: + requests: + memory: 256Mi + jvmXmx: 256M +## Configuration for renku-graph services graph: webhookService: aesEncryptionKey: # A 8, 16 or 32 bytes string used for AES encryption of the project tokens diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml index 393c415139..c630909dbf 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml @@ -27,7 +27,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password ports: - containerPort: 5432 volumeMounts: @@ -81,7 +81,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml index d4395019f2..1ad02e0d18 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml @@ -28,7 +28,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /bitnami/postgresql name: pg-vol-new @@ -80,7 +80,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index 41fdf5ee1d..2a38c21660 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,47 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.xx.0 +* NEW/EDIT - *postgresql.persistence.existingClaim*: Renku `0.xx.0` upgrades the postgres chart dependency, which requires modification of the postgres data volume of existing deployments. See [these instructions](https://github.com/SwissDataScienceCenter/renku/tree/master/helm-chart/utils/postgres_migrations/version_upgrades/README.md) +* EDIT - *postgresql*: The upgrade of the postgres chart dependency requires some restructuring of the postgres subchart values to match those of bitnami/postgresql chart version 12.6.8, namely: + +Old + ``` + postgresql: + postgresqlDatabase: + postgresqlUsername: + postgresqlPassword: + existingSecret: + persistence: + enabled: + size: + existingClaim: + replication: + enabled: + user: + password: + slaveReplicas: + ``` +New + ``` + postgresql: + auth: + username: + database: + postgresqlPassword: + existingSecret: + replicationUsername: + replicationPassword: + primary: + persistence: + enabled: + size: + existingClaim: + readReplicas: + enabled: + replicaCount: + ``` + ## Upgrading to Renku 0.48.1 The handling of privacy policy and terms of service content has been fine tuned. From 2c652a9f80a2929f5f493053cb3bd559906eef8e Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 19 Feb 2024 10:58:14 +0100 Subject: [PATCH 05/61] chore: enabling solr on the branch --- helm-chart/renku/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index a3a36a3ce4..f176ec7dc3 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -482,7 +482,7 @@ redis: sysctl: enabled: false solr: - enabled: false + enabled: true cloudEnabled: false auth: adminUsername: renku From cb35487ff0a4e4e54e274ab081e9836ad1abfeff Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 19 Feb 2024 17:11:53 +0100 Subject: [PATCH 06/61] chore: search upgraded to 0.0.12 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index f176ec7dc3..3ef3ac4d4f 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1334,7 +1334,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.11" + tag: "0.0.12" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1347,7 +1347,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.11" + tag: "0.0.12" pullPolicy: IfNotPresent service: type: ClusterIP From 5f28d157b93b91a7e848c6c0b7a33bea880b3896 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 19 Feb 2024 17:44:21 +0100 Subject: [PATCH 07/61] chore: no replicas for Solr --- helm-chart/renku/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 3ef3ac4d4f..e85b499704 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -484,6 +484,7 @@ redis: solr: enabled: true cloudEnabled: false + collectionReplicas: 0 auth: adminUsername: renku adminPassword: renku From 6d3da0af1e2dbcf9a71338784c112e414b6c74eb Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 19 Feb 2024 17:51:39 +0100 Subject: [PATCH 08/61] chore: no replicas for Solr --- helm-chart/renku/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index e85b499704..8073625d9a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -485,6 +485,7 @@ solr: enabled: true cloudEnabled: false collectionReplicas: 0 + replicaCount: 0 auth: adminUsername: renku adminPassword: renku From d304fdbe2b9bfa999bb19acd13cc59bd195cfaf7 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 19 Feb 2024 17:57:40 +0100 Subject: [PATCH 09/61] chore: 1 replica for Solr --- helm-chart/renku/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 8073625d9a..7258765ad9 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -485,7 +485,7 @@ solr: enabled: true cloudEnabled: false collectionReplicas: 0 - replicaCount: 0 + replicaCount: 1 auth: adminUsername: renku adminPassword: renku From e0577d9f6602e9736445096d06f993a79a10624a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 20 Feb 2024 17:38:22 +0100 Subject: [PATCH 10/61] chore: reducing max heap space to 152M on Solr --- helm-chart/renku/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 7258765ad9..adc2dbcb3f 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -491,6 +491,7 @@ solr: adminPassword: renku coreNames: - renku-search + javaMem: "-Xmx512M" networkPolicy: enabled: false # Awaiting instructions from: https://github.com/bitnami/charts/issues/23424 persistence: From 6fc5f15d9d5e1aa2a60b1b863193bd3fa8bf5f82 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 21 Feb 2024 14:30:18 +0100 Subject: [PATCH 11/61] chore: search swagger.yaml linked to global renku swagger --- helm-chart/renku/templates/swagger.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/renku/templates/swagger.yaml b/helm-chart/renku/templates/swagger.yaml index 370b6660da..c23a3a9f28 100644 --- a/helm-chart/renku/templates/swagger.yaml +++ b/helm-chart/renku/templates/swagger.yaml @@ -30,6 +30,7 @@ spec: {"url": "/api/kg/spec.json", "name": "knowledge graph"}, {"url": "/api/notebooks/spec.json", "name": "notebooks service"}, {"url": "/api/data/spec.json", "name": "data service"} + {"url": "/search/docs/docs.yaml", "name": "search service"} ] - name: OAUTH2_REDIRECT_URL value: https://{{ .Values.global.renku.domain }}/swagger/oauth2-redirect.html From 3f9c2f14af56ebf5d9a534dcad270b44c832d82e Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 21 Feb 2024 14:33:42 +0100 Subject: [PATCH 12/61] chore: search services upgraded to 0.0.13 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index adc2dbcb3f..964eee1967 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.12" + tag: "0.0.13" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.12" + tag: "0.0.13" pullPolicy: IfNotPresent service: type: ClusterIP From 1286612f3c31af8c895e8b7758d086114375b43a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 21 Feb 2024 15:52:28 +0100 Subject: [PATCH 13/61] chore: search services upgraded to 0.0.14 and fix for the renku swagger page --- helm-chart/renku/templates/swagger.yaml | 2 +- helm-chart/renku/values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/renku/templates/swagger.yaml b/helm-chart/renku/templates/swagger.yaml index c23a3a9f28..f621fe574e 100644 --- a/helm-chart/renku/templates/swagger.yaml +++ b/helm-chart/renku/templates/swagger.yaml @@ -30,7 +30,7 @@ spec: {"url": "/api/kg/spec.json", "name": "knowledge graph"}, {"url": "/api/notebooks/spec.json", "name": "notebooks service"}, {"url": "/api/data/spec.json", "name": "data service"} - {"url": "/search/docs/docs.yaml", "name": "search service"} + {"url": "/search/spec.json", "name": "search service"} ] - name: OAUTH2_REDIRECT_URL value: https://{{ .Values.global.renku.domain }}/swagger/oauth2-redirect.html diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 964eee1967..69275fa035 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.13" + tag: "0.0.14" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.13" + tag: "0.0.14" pullPolicy: IfNotPresent service: type: ClusterIP From 7341ca0545ccc0a3fba448e63acd6a1f24fc79c4 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 21 Feb 2024 15:58:06 +0100 Subject: [PATCH 14/61] chore: typo in the swagger.yaml --- helm-chart/renku/templates/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/templates/swagger.yaml b/helm-chart/renku/templates/swagger.yaml index f621fe574e..1ee5133ab1 100644 --- a/helm-chart/renku/templates/swagger.yaml +++ b/helm-chart/renku/templates/swagger.yaml @@ -29,7 +29,7 @@ spec: {"url": "/api/renku/spec.json", "name": "core service"}, {"url": "/api/kg/spec.json", "name": "knowledge graph"}, {"url": "/api/notebooks/spec.json", "name": "notebooks service"}, - {"url": "/api/data/spec.json", "name": "data service"} + {"url": "/api/data/spec.json", "name": "data service"}, {"url": "/search/spec.json", "name": "search service"} ] - name: OAUTH2_REDIRECT_URL From 36763c65544fda07362b828f8549a7e9ee5ea560 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Thu, 22 Feb 2024 14:54:18 +0100 Subject: [PATCH 15/61] chore: search services upgraded to 0.0.15 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 69275fa035..c013d900e2 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.14" + tag: "0.0.15" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.14" + tag: "0.0.15" pullPolicy: IfNotPresent service: type: ClusterIP From c5d79a80928b5629e9f5adaf41ea127335983e1b Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 15:02:19 +0100 Subject: [PATCH 16/61] chore: search services upgraded to 0.0.16 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index c013d900e2..dfa9ab3516 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.15" + tag: "0.0.16" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.15" + tag: "0.0.16" pullPolicy: IfNotPresent service: type: ClusterIP From 182099719d6cde0ecec343592e4897b6b2bb8347 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 15:28:42 +0100 Subject: [PATCH 17/61] chore: search services upgraded to 0.0.17 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index dfa9ab3516..d133668ced 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.16" + tag: "0.0.17" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.16" + tag: "0.0.17" pullPolicy: IfNotPresent service: type: ClusterIP From 4b3daeac125e2ec2772bead971db65a5776e05c4 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 15:40:19 +0100 Subject: [PATCH 18/61] chore: search services upgraded to 0.0.18 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index d133668ced..97803b93a6 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.17" + tag: "0.0.18" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.17" + tag: "0.0.18" pullPolicy: IfNotPresent service: type: ClusterIP From c8832e37115015754ae38b5771e192ad1f710584 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 26 Feb 2024 16:31:28 +0100 Subject: [PATCH 19/61] chore: remove changelog conflicts --- CHANGELOG.rst | 62 --------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be3a3443f0..3c37c9b94e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,66 +1,4 @@ .. _changelog: -0.xx.0 ------- - -Renku ``0.xx.0`` is a release that exclusively changes internal components and has no effect on the end-user. -Upgrades the PostgreSQL Helm chart version that is shipped with the Renku Helm chart. - -NOTE for administrators: make sure to check `helm-chart/values.yaml.changelog.md` for instructions on how to -upgrade to this version of Renku. -**Improvements** - -- **Infrastructure**: (BREAKING CHANGE) Upgrade the version of PostgreSQL to 16.1.0. This requires modifying your current values file to work with the new PostgreSQL Helm chart, see (`the helm chart values changelog `_) for instructions. - -0.48.0 ------- - -User-Facing Changes -~~~~~~~~~~~~~~~~~~~ - -**🌟 New Features** - -- 📜 **UI**: Show terms of use and privacy policy in the help section - (`#2954 `_). - -**✨ Improvements** - -- 🖌 **UI**: Improve appearance of templates on new project page - (`#2999 `_). -- 🛑 **UI**: Unify appearance of project settings alerts - (`#3001 `_). - -**🐞 Bug Fixes** - -- **UI**: Restore logged in/out notifications - (`#3014 `_). -- **UI**: Hide button to add storage on deployments not supporting external storages - (`#3001 `_). - -Internal Changes -~~~~~~~~~~~~~~~~ - -**Improvements** - -- **UI**: Add initial alpha implementation of Renku 1.0 projects - (`#2875 `_). - -Individual components -~~~~~~~~~~~~~~~~~~~~~~ - -- `renku-ui 3.20.0 `_ - - -0.49.0 ------- - -TODO: Search & Discovery 1.0 - - -0.48.1 ------- - -Renku ``0.48.1`` only changes how the Terms of Use and Privacy Policy sections -can be customized by administrators. 0.48.1 ------ From 2a7e7f277a66509f28ef2586b8592b757819f9dc Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 26 Feb 2024 17:05:19 +0100 Subject: [PATCH 20/61] add message queue redis to data services (#3508) --- helm-chart/renku/templates/_helpers.tpl | 4 + .../templates/data-service/deployment.yaml | 23 +++ .../keycloak-users-sync-cronjob.yaml | 113 ++++++++++++++ .../renku/templates/network-policies.yaml | 6 + helm-chart/renku/templates/secrets.yaml | 10 ++ .../templates/setup-job-keycloak-realms.yaml | 23 +++ helm-chart/renku/values.yaml | 5 + scripts/init-realm/Dockerfile | 2 +- scripts/init-realm/init-realm.py | 87 +++++++++-- scripts/init-realm/utils.py | 145 ++++++++++++++---- 10 files changed, 371 insertions(+), 47 deletions(-) create mode 100644 helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl index fb71c7d9ba..9dedf80850 100644 --- a/helm-chart/renku/templates/_helpers.tpl +++ b/helm-chart/renku/templates/_helpers.tpl @@ -168,3 +168,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- define "renku.keycloak.realm" -}} {{ .Values.global.keycloak.realm | default "Renku" }} {{- end -}} + +{{- define "renku.dataService.keycloak.clientId" -}} +data-service +{{- end -}} diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index bed53c5f51..470ff52d5b 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -22,6 +22,7 @@ spec: labels: app: renku-data-service release: {{ .Release.Name }} + {{ .Values.global.redis.clientLabel | toYaml | nindent 8 }} {{- with .Values.dataService.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} @@ -58,6 +59,13 @@ spec: value: {{ (printf "%s://%s/auth/" (include "renku.http" .) .Values.global.renku.domain) | quote }} - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS value: "RS256" + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret - name: SERVER_DEFAULTS value: /etc/renku-data-service/server_options/server_defaults.json - name: SERVER_OPTIONS @@ -68,6 +76,21 @@ spec: value: {{ .Values.global.gitlab.url | quote }} - name: MAX_PINNED_PROJECTS value: {{ .Values.dataService.maxPinnedProjects | quote }} + - name: REDIS_HOST + value: {{ .Values.global.redis.host | quote }} + - name: REDIS_PORT + value: {{ .Values.global.redis.port | quote }} + - name: REDIS_DATABASE + value: {{ .Values.global.redis.dbIndex.events| quote }} + - name: REDIS_IS_SENTINEL + value: {{ .Values.global.redis.sentinel.enabled | quote }} + - name: REDIS_MASTER_SET + value: {{ .Values.global.redis.sentinel.masterSet | quote }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: {{ .Values.global.redis.existingSecretPasswordKey }} {{- include "certificates.env.python" $ | nindent 12 }} volumeMounts: - name: server-options diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml new file mode 100644 index 0000000000..c109f4875a --- /dev/null +++ b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml @@ -0,0 +1,113 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "renku.fullname" . }}-keycloak-sync-events + labels: + app: keycloak-sync + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "renku.chart" . }} +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + app: keycloak-sync + spec: + initContainers: + {{- include "certificates.initContainer" . | nindent 12 }} + containers: + - name: keycloak-sync + image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}" + imagePullPolicy: IfNotPresent + env: + - name: DB_HOST + value: {{ template "postgresql.fullname" . }} + - name: DB_USER + value: {{ .Values.global.db.common.username }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.common.passwordSecretName }} + key: password + - name: KEYCLOAK_URL + value: {{ include "renku.keycloakUrl" . | quote}} + - name: KEYCLOAK_REALM + value: {{ include "renku.keycloak.realm" . | quote}} + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: TOTAL_USER_SYNC + value: "false" + {{- include "certificates.env.python" . | nindent 16 }} + volumeMounts: + {{- include "certificates.volumeMounts.system" . | nindent 16 }} + restartPolicy: Never + volumes: + {{- include "certificates.volumes" . | nindent 12 }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "renku.fullname" . }}-keycloak-sync-total + labels: + app: keycloak-sync + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "renku.chart" . }} +spec: + schedule: "0 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + app: keycloak-sync + spec: + initContainers: + {{- include "certificates.initContainer" . | nindent 12 }} + containers: + - name: keycloak-sync + image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}" + imagePullPolicy: IfNotPresent + env: + - name: DB_HOST + value: {{ template "postgresql.fullname" . }} + - name: DB_USER + value: {{ .Values.global.db.common.username }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.common.passwordSecretName }} + key: password + - name: KEYCLOAK_URL + value: {{ include "renku.keycloakUrl" . | quote}} + - name: KEYCLOAK_REALM + value: {{ include "renku.keycloak.realm" . | quote}} + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: TOTAL_USER_SYNC + value: "true" + {{- include "certificates.env.python" . | nindent 16 }} + volumeMounts: + {{- include "certificates.volumeMounts.system" . | nindent 16 }} + restartPolicy: Never + volumes: + {{- include "certificates.volumes" . | nindent 12 }} diff --git a/helm-chart/renku/templates/network-policies.yaml b/helm-chart/renku/templates/network-policies.yaml index b2c77647ef..431f5fbd6d 100644 --- a/helm-chart/renku/templates/network-policies.yaml +++ b/helm-chart/renku/templates/network-policies.yaml @@ -69,6 +69,12 @@ spec: namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} + - podSelector: + matchLabels: + app: keycloak-sync + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} ports: - protocol: TCP port: 5432 diff --git a/helm-chart/renku/templates/secrets.yaml b/helm-chart/renku/templates/secrets.yaml index 0b39dfac76..c9b8a42110 100644 --- a/helm-chart/renku/templates/secrets.yaml +++ b/helm-chart/renku/templates/secrets.yaml @@ -1,4 +1,11 @@ --- +{{- $data_service_kc_client_secret := randAlphaNum 64 | b64enc | quote }} + +{{- $renku_secret := lookup "v1" "Secret" .Release.Namespace (include "renku.fullname" .) }} +{{- if $renku_secret }} +{{- $data_service_kc_client_secret = index $renku_secret.data "dataServiceKeycloakClientSecret" }} +{{- end -}} + apiVersion: v1 kind: Secret metadata: @@ -8,11 +15,14 @@ metadata: chart: {{ template "renku.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": "pre-install,pre-upgrade,pre-rollback" type: Opaque data: {{- if .Values.tests.users_json }} users.json: {{ .Values.tests.users_json | toJson | b64enc | quote }} {{- end }} + dataServiceKeycloakClientSecret: {{ $data_service_kc_client_secret }} {{- if and (eq .Values.redis.install true) (eq .Values.redis.createSecret true) }} diff --git a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml index 0efc5d88fc..84735d221c 100644 --- a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml +++ b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml @@ -86,6 +86,8 @@ spec: key: oidcClientSecret - name: RENKU_KC_CLIENT_PUBLIC value: "false" + - name: RENKU_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: CLI_KC_CLIENT_ID value: renku-cli - name: CLI_KC_CLIENT_SECRET @@ -97,6 +99,8 @@ spec: value: "true" - name: CLI_KC_CLIENT_ATTRIBUTES value: '{"access.token.lifespan": "86400", "oauth2.device.authorization.grant.enabled": true, "oauth2.device.polling.interval": "5"}' + - name: CLI_KC_CLIENT_OAUTH_FLOW + value: "device" - name: UI_KC_CLIENT_ID value: "renku-ui" - name: UI_KC_CLIENT_SECRET @@ -106,6 +110,8 @@ spec: key: uiserverClientSecret - name: UI_KC_CLIENT_PUBLIC value: "false" + - name: UI_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: NOTEBOOKS_KC_CLIENT_ID value: {{ .Values.notebooks.oidc.clientId | default "renku-jupyterserver" | quote }} - name: NOTEBOOKS_KC_CLIENT_SECRET @@ -115,12 +121,29 @@ spec: key: notebooksClientSecret - name: NOTEBOOKS_KC_CLIENT_PUBLIC value: "false" + - name: NOTEBOOKS_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: SWAGGER_KC_CLIENT_ID value: swagger - name: SWAGGER_KC_CLIENT_PUBLIC value: "true" + - name: SWAGGER_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: SWAGGER_KC_CLIENT_ATTRIBUTES value: '{"pkce.code.challenge.method": "S256"}' + - name: DATASERVICE_KC_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: DATASERVICE_KC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: DATASERVICE_KC_CLIENT_PUBLIC + value: "false" + - name: DATASERVICE_KC_CLIENT_OAUTH_FLOW + value: "client_credentials" + - name: "DATASERVICE_KC_CLIENT_SERVICE_ACCOUNT_ROLES" + value: '["view-users", "query-users", "view-events"]' - name: PYTHONUNBUFFERED value: "0" {{- include "certificates.env.python" . | nindent 12 }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 97803b93a6..df529851ae 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1545,6 +1545,11 @@ dataService: repository: renku/renku-data-service tag: "0.4.0" pullPolicy: IfNotPresent + keycloakSync: + image: + repository: renku/keycloak-sync + tag: "0.2.2" + pullPolicy: IfNotPresent service: type: ClusterIP port: 80 diff --git a/scripts/init-realm/Dockerfile b/scripts/init-realm/Dockerfile index 2fa09059cf..8ed86a2417 100644 --- a/scripts/init-realm/Dockerfile +++ b/scripts/init-realm/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine COPY requirements.txt init-realm.py utils.py /app/ WORKDIR /app diff --git a/scripts/init-realm/init-realm.py b/scripts/init-realm/init-realm.py index ea784e6a9c..ec3bac706c 100644 --- a/scripts/init-realm/init-realm.py +++ b/scripts/init-realm/init-realm.py @@ -21,6 +21,7 @@ import json import time import logging +import os from typing import Dict, List from keycloak import KeycloakAdmin @@ -30,7 +31,7 @@ KeycloakPostError, ) -from utils import DemoUserConfig, OIDCClientsConfig, OIDCGitlabClient +from utils import DemoUserConfig, OIDCClientsConfig, OIDCGitlabClient, OIDCClient, OAuthFlow logging.basicConfig(level=logging.INFO) @@ -70,18 +71,19 @@ def _fix_json_values(data: Dict) -> Dict: return json.loads(json.dumps(data).replace('"true"', "true").replace('"false"', "false")) -def _check_and_create_client(keycloak_admin, new_client, force: bool): +def _check_and_create_client(keycloak_admin, new_client: OIDCClient, force: bool): """ Check if a client exists. Create it if not. Alert if it exists but with different details than what is provided. """ - logging.info("Checking if {} client exists...".format(new_client["clientId"])) + logging.info("Checking if {} client exists...".format(new_client.id)) realm_clients = keycloak_admin.get_clients() client_ids = [c["clientId"] for c in realm_clients] - if new_client["clientId"] in client_ids: + realm_management_client_id = keycloak_admin.get_client_id("realm-management") + if new_client.id in client_ids: logging.info("found") - realm_client = realm_clients[client_ids.index(new_client["clientId"])] + realm_client = realm_clients[client_ids.index(new_client.id)] # We have to separately query the secret as it is not part of # the original response @@ -99,22 +101,57 @@ def _check_and_create_client(keycloak_admin, new_client, force: bool): if "attributes" in realm_client: realm_client["attributes"] = _fix_json_values(realm_client["attributes"]) - changed = _check_existing(realm_client, new_client, "client", "clientId") - - if not force or not changed: + roles_changed = False + service_account_user = None + existing_roles = [] + if new_client.oauth_flow == OAuthFlow.client_credentials: + try: + service_account_user = keycloak_admin.get_client_service_account_user(realm_client["id"]) + except KeycloakGetError as err: + if err.response_code != 404: + raise + if isinstance(service_account_user, dict): + try: + existing_roles = keycloak_admin.get_client_roles_of_user(service_account_user["id"], realm_management_client_id) + except KeycloakGetError as err: + if err.response_code != 404: + raise + existing_roles_names = [role["name"] for role in existing_roles] + if set(existing_roles_names) != set(new_client.service_account_roles): + logging.warning(f"Roles changed existing roles {set(existing_roles_names)} != new roles {set(new_client.service_account_roles)}") + roles_changed = True + changed = _check_existing(realm_client, new_client.to_dict(), "client", "clientId") + + if not force or (not changed and not roles_changed): return logging.info(f"Recreating modified client '{realm_client['clientId']}'...") keycloak_admin.delete_client(realm_client["id"]) - keycloak_admin.create_client(new_client) + created_client_id = keycloak_admin.create_client(new_client.to_dict()) + + if isinstance(service_account_user, dict) and service_account_user.get("id"): + logging.info(f"Reassigning service account roles {new_client.service_account_roles}") + realm_management_roles = keycloak_admin.get_client_roles(realm_management_client_id) + matching_roles = [{"name": role["name"], "id": role["id"]} for role in realm_management_roles if role["name"] in new_client.service_account_roles ] + logging.info(f"Found and assigning matching roles: {matching_roles}") + keycloak_admin.assign_client_role(service_account_user["id"], realm_management_client_id, matching_roles) logging.info("done") else: logging.info("not found") - logging.info("Creating {} client...".format(new_client["clientId"])) - keycloak_admin.create_client(new_client) + logging.info("Creating {} client...".format(new_client.id)) + created_client_id = keycloak_admin.create_client(new_client.to_dict()) + if new_client.oauth_flow == OAuthFlow.client_credentials and new_client.service_account_roles: + service_account_user = keycloak_admin.get_client_service_account_user(created_client_id) + logging.info(f"Assigning service account roles {new_client.service_account_roles}") + realm_management_client_id = keycloak_admin.get_client_id("realm-management") + realm_management_roles = keycloak_admin.get_client_roles(realm_management_client_id) + matching_roles = [{"name": role["name"], "id": role["id"]} for role in realm_management_roles if role["name"] in new_client.service_account_roles ] + logging.info(f"Found and assigning matching roles: {matching_roles}") + keycloak_admin.assign_client_role(service_account_user["id"], realm_management_client_id, matching_roles) + logging.info("done") @@ -234,15 +271,35 @@ def _check_and_create_user(keycloak_admin, new_user): ) logging.info("done") +realm = keycloak_admin.get_realm(args.realm) +event_retention_seconds = 86400 +if not realm.get("eventsEnabled"): + logging.info( + f"Enabling user events tracking for realm with retention {event_retention_seconds}" + ) + keycloak_admin.update_realm(args.realm, {"eventsEnabled": True, "eventsExpiration": event_retention_seconds}) +if not realm.get("adminEventsEnabled"): + logging.info( + f"Enabling admin events tracking for realm with retention {event_retention_seconds}" + ) + keycloak_admin.update_realm( + args.realm, + { + "adminEventsEnabled": True, + "adminEventsDetailsEnabled": True, + "attributes": {"adminEventsExpiration": event_retention_seconds}, + }, + ) + # Switching to the newly created realm keycloak_admin.connection.realm_name = args.realm -for new_client in OIDCClientsConfig.from_env().to_list(): - _check_and_create_client(keycloak_admin, new_client, args.force) +for client in OIDCClientsConfig.from_env().to_list(): + _check_and_create_client(keycloak_admin, client, args.force) -gitlab_oidc_client = OIDCGitlabClient.from_env().to_dict() -if gitlab_oidc_client is not None: +if os.environ.get("INTERNAL_GITLAB_ENABLED", "false").lower() == "true": + gitlab_oidc_client = OIDCGitlabClient.from_env() _check_and_create_client(keycloak_admin, gitlab_oidc_client, args.force) # Create renku-admin realm role diff --git a/scripts/init-realm/utils.py b/scripts/init-realm/utils.py index ff009e5930..63501d58bf 100644 --- a/scripts/init-realm/utils.py +++ b/scripts/init-realm/utils.py @@ -1,6 +1,8 @@ import json import os +from copy import deepcopy from dataclasses import dataclass, field +from enum import Enum from typing import Any, Dict, List, Optional @@ -36,6 +38,39 @@ def to_dict(self) -> Optional[Dict[str, Any]]: } +class OAuthFlow(Enum): + device: str = "device" + authorization_code: str = "authorization_code" + client_credentials: str = "client_credentials" + + def get_keycloak_payload( + self, + existing_payload: Dict[str, Any] | None = None, + disable_other_flows: bool = True + ) -> Dict[str, Any]: + output = deepcopy(existing_payload) if existing_payload else {} + if disable_other_flows: + output.update( + serviceAccountsEnabled=False, + standardFlowEnabled=False, + ) + match self: + case OAuthFlow.authorization_code: + output["standardFlowEnabled"] = True + case OAuthFlow.device: + if isinstance(output.get("attributes"), dict): + output["attributes"]["oauth2.device.authorization.grant.enabled"] = True + else: + output["attributes"] = {"oauth2.device.authorization.grant.enabled": True} + case OAuthFlow.client_credentials: + output["serviceAccountsEnabled"] = True + return output + + @classmethod + def from_env(cls, prefix: str = ""): + return cls(os.environ.get(f"{prefix}OAUTH_FLOW")) + + @dataclass class OIDCClient: """Stores the configuration needed to create an OIDC client application in Keycloak. These @@ -44,8 +79,11 @@ class OIDCClient: id: str base_url: str + oauth_flow: OAuthFlow + disable_other_oauth_flows: bool = True secret: Optional[str] = field(default=None, repr=False) attributes: Dict[str, Any] = field(default_factory=lambda: {}) + service_account_roles: List[str] = field(default_factory=list) public_client: bool = False def __post_init__(self): @@ -55,8 +93,58 @@ def __post_init__(self): f"The OIDC client configuration for client {self.id} is not valid, " "the client is marked as not public but a secret is not provided." ) + if self.oauth_flow != OAuthFlow.client_credentials and len(self.service_account_roles) > 0: + raise ValueError( + f"Service account roles can only be specified for the {OAuthFlow.client_credentials.value} flow" + ) def to_dict(self) -> Dict[str, Any]: + default_protocol_mappers = [] + if self.oauth_flow == OAuthFlow.client_credentials: + default_protocol_mappers.extend([ + { + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientId", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientHost", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientHost", + "jsonType.label": "String" + } + }, + { + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientAddress", + "jsonType.label": "String" + } + } + ]) output = { "clientId": self.id, "baseUrl": self.base_url, @@ -64,7 +152,7 @@ def to_dict(self) -> Dict[str, Any]: "attributes": self.attributes, "redirectUris": [self.base_url + "/*"], "webOrigins": [self.base_url + "/*"], - "protocolMappers": [ + "protocolMappers": default_protocol_mappers + [ { "name": "renku audience for renku cli", "protocol": "openid-connect", @@ -81,6 +169,7 @@ def to_dict(self) -> Dict[str, Any]: } if self.secret is not None: output["secret"] = self.secret + output = self.oauth_flow.get_keycloak_payload(output, self.disable_other_oauth_flows) return output @classmethod @@ -91,43 +180,34 @@ def from_env(cls, prefix: str = "RENKU_KC_CLIENT_") -> "OIDCClient": base_url=os.environ.get(f"{prefix}BASE_URL", os.environ["RENKU_BASE_URL"]), attributes=json.loads(os.environ.get(f"{prefix}ATTRIBUTES", "{}")), public_client=os.environ.get(f"{prefix}PUBLIC", "false").lower() == "true", + oauth_flow=OAuthFlow.from_env(prefix), + disable_other_oauth_flows=os.environ.get( + f"{prefix}DISABLE_OTHER_OAUTH_FLOWS", "true" + ).lower() == "true", + service_account_roles=json.loads(os.environ.get(f"{prefix}SERVICE_ACCOUNT_ROLES", "[]")), ) @dataclass -class OIDCGitlabClient: - """A Keycloak OIDC client used by the internal Renku Gitlab deployment (if this deployment is enabled).""" - - internal_gitlab_enabled: bool = False - oidc_client_secret: Optional[str] = field(default=None, repr=False) - oidc_client_id: str = "gitlab" - renku_base_url: Optional[str] = None - - def __post_init__(self): - if self.internal_gitlab_enabled and not (self.oidc_client_secret or self.renku_base_url): - raise ValueError( - "The internal Gitlab is enabled, but the Renku base URL and/or the Keycloak OIDC client secret are not defined." - ) - self.renku_base_url = self.renku_base_url.rstrip("/") +class OIDCGitlabClient(OIDCClient): + """A Keycloak OIDC client used by the internal Renku Gitlab deployment.""" @classmethod def from_env(cls, prefix: str = "INTERNAL_GITLAB_") -> "OIDCGitlabClient": return cls( - internal_gitlab_enabled=os.environ.get(f"{prefix}ENABLED", "false").lower() == "true", - oidc_client_secret=os.environ.get(f"{prefix}OIDC_CLIENT_SECRET"), - oidc_client_id=os.environ.get(f"{prefix}OIDC_CLIENT_ID", "gitlab"), - renku_base_url=os.environ.get(f"RENKU_BASE_URL"), + secret=os.environ.get(f"{prefix}OIDC_CLIENT_SECRET"), + id=os.environ.get(f"{prefix}OIDC_CLIENT_ID", "gitlab"), + base_url=os.environ.get("RENKU_BASE_URL"), + oauth_flow=OAuthFlow.authorization_code, ) def to_dict(self) -> Optional[Dict[str, Any]]: - if not self.internal_gitlab_enabled: - return None return { - "clientId": self.oidc_client_id, - "baseUrl": f"{self.renku_base_url}/gitlab", - "secret": self.oidc_client_secret, + "clientId": self.id, + "baseUrl": f"{self.base_url}", + "secret": self.secret, "redirectUris": [ - f"{self.renku_base_url}/gitlab/users/auth/oauth2_generic/callback", + f"{self.base_url}/users/auth/oauth2_generic/callback", ], "webOrigins": [], } @@ -140,6 +220,7 @@ class OIDCClientsConfig: ui: OIDCClient notebooks: OIDCClient swagger: OIDCClient + data_service: OIDCClient @classmethod def from_env(cls) -> "OIDCClientsConfig": @@ -149,13 +230,15 @@ def from_env(cls) -> "OIDCClientsConfig": ui=OIDCClient.from_env(prefix="UI_KC_CLIENT_"), notebooks=OIDCClient.from_env(prefix="NOTEBOOKS_KC_CLIENT_"), swagger=OIDCClient.from_env(prefix="SWAGGER_KC_CLIENT_"), + data_service=OIDCClient.from_env(prefix="DATASERVICE_KC_CLIENT_"), ) - def to_list(self) -> List[Dict[str, Any]]: + def to_list(self) -> List[OIDCClient]: return [ - self.renku.to_dict(), - self.cli.to_dict(), - self.ui.to_dict(), - self.notebooks.to_dict(), - self.swagger.to_dict(), + self.renku, + self.cli, + self.ui, + self.notebooks, + self.swagger, + self.data_service, ] From 59caabbf13fcd5b11b109d2340c6cea2854efbb4 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 26 Feb 2024 17:41:43 +0100 Subject: [PATCH 21/61] chore: solr upgraded to 8.9.2 --- helm-chart/renku/requirements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 67baafd94a..a7bba2dade 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -34,5 +34,5 @@ dependencies: condition: global.csi-rclone.install - name: solr repository: "oci://registry-1.docker.io/bitnamicharts" - version: "8.7.1" + version: "8.9.2" condition: solr.enabled From 78ca843daac45d0c6fe19026aab207189c1ce5d1 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 27 Feb 2024 18:24:10 +0100 Subject: [PATCH 22/61] chore: search upgraded to 0.0.19 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index df529851ae..a0d49a1cd9 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.18" + tag: "0.0.19" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.18" + tag: "0.0.19" pullPolicy: IfNotPresent service: type: ClusterIP From d87150665b1f84fe2453544d4dd78ce4b5c2bf9f Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 27 Feb 2024 18:45:25 +0100 Subject: [PATCH 23/61] chore: search upgraded to 0.0.19 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index a0d49a1cd9..4d628d18ef 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.19" + tag: "b08a4a3664fc" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.19" + tag: "b08a4a3664fc" pullPolicy: IfNotPresent service: type: ClusterIP From 7bf0fbdaaaac267f127470c38bf5429c0d140cfa Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 28 Feb 2024 07:27:16 +0100 Subject: [PATCH 24/61] chore: search upgraded to 0.0.21 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 4d628d18ef..5735d332b2 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "b08a4a3664fc" + tag: "0.0.21" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "b08a4a3664fc" + tag: "0.0.21" pullPolicy: IfNotPresent service: type: ClusterIP From 68a29bc541f09f935614e84ffa1a5979df3963f3 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 28 Feb 2024 10:25:45 +0100 Subject: [PATCH 25/61] chore: match the search api root path on ingress and swagger --- helm-chart/renku/templates/ingress.yaml | 2 +- helm-chart/renku/templates/swagger.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/templates/ingress.yaml b/helm-chart/renku/templates/ingress.yaml index d82240c1e7..fb9c9064c0 100644 --- a/helm-chart/renku/templates/ingress.yaml +++ b/helm-chart/renku/templates/ingress.yaml @@ -127,7 +127,7 @@ spec: port: number: 80 {{- end }} - - path: /search + - path: /apiv2/search pathType: Prefix backend: service: diff --git a/helm-chart/renku/templates/swagger.yaml b/helm-chart/renku/templates/swagger.yaml index 1ee5133ab1..38293f5cdc 100644 --- a/helm-chart/renku/templates/swagger.yaml +++ b/helm-chart/renku/templates/swagger.yaml @@ -30,7 +30,7 @@ spec: {"url": "/api/kg/spec.json", "name": "knowledge graph"}, {"url": "/api/notebooks/spec.json", "name": "notebooks service"}, {"url": "/api/data/spec.json", "name": "data service"}, - {"url": "/search/spec.json", "name": "search service"} + {"url": "/apiv2/search/spec.json", "name": "search service"} ] - name: OAUTH2_REDIRECT_URL value: https://{{ .Values.global.renku.domain }}/swagger/oauth2-redirect.html From 9189cdf1de86077b3c4055810a18b977299347f0 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 28 Feb 2024 13:25:48 +0100 Subject: [PATCH 26/61] chore: search upgraded to 0.0.22 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 5735d332b2..6db7f80cb2 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.21" + tag: "0.0.22" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.21" + tag: "0.0.22" pullPolicy: IfNotPresent service: type: ClusterIP From 1f1ed1a82b632b770885fd60af8c3c37dd69b239 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Thu, 29 Feb 2024 11:42:23 +0100 Subject: [PATCH 27/61] chore: search upgraded to 0.0.23 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 6db7f80cb2..9cf93c6790 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.22" + tag: "0.0.23" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.22" + tag: "0.0.23" pullPolicy: IfNotPresent service: type: ClusterIP From ba979581b0e9c03040787b078669236e07fd4a2f Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 5 Mar 2024 12:43:50 +0100 Subject: [PATCH 28/61] chore: improved config of event streams config --- .../renku/templates/search/search-provision-deployment.yaml | 6 ++++-- helm-chart/renku/values.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml index a227df315d..9304321144 100644 --- a/helm-chart/renku/templates/search/search-provision-deployment.yaml +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -48,8 +48,10 @@ spec: key: {{ .Values.global.redis.existingSecretPasswordKey }} - name: RS_REDIS_MASTER_SET value: {{ .Values.global.redis.sentinel.masterSet | quote }} - - name: RS_REDIS_QUEUE_NAME - value: {{ first .Values.global.events.streams }} + {{ - range $k, $v := .Values.global.events.streams }} + - name: RS_REDIS_QUEUE_{{ $k }} + value: {{ $v }} + {{ - end }} - name: RS_SOLR_URL value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr" - name: RS_SOLR_CORE diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 9cf93c6790..4d1ee16f69 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -82,7 +82,7 @@ global: ## Events related values definitions events: streams: - - project.created + projectCreated: "project.created" ## Note that the graph will not turned on by default until renku 0.4.0 graph: dbEventLog: From f8d0aae56063239be39b4dcfa3a8381babf808cb Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 5 Mar 2024 12:44:12 +0100 Subject: [PATCH 29/61] chore: search services upgraded to 0.0.24 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 4d1ee16f69..79a4fa3d8c 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.23" + tag: "0.0.24" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.23" + tag: "0.0.24" pullPolicy: IfNotPresent service: type: ClusterIP From b75271161132fddee2710b4efbb6291ad369f823 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 5 Mar 2024 12:47:14 +0100 Subject: [PATCH 30/61] chore: typo in the search-provision-deployment.yaml --- .../renku/templates/search/search-provision-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml index 9304321144..bae9b89b13 100644 --- a/helm-chart/renku/templates/search/search-provision-deployment.yaml +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -48,10 +48,10 @@ spec: key: {{ .Values.global.redis.existingSecretPasswordKey }} - name: RS_REDIS_MASTER_SET value: {{ .Values.global.redis.sentinel.masterSet | quote }} - {{ - range $k, $v := .Values.global.events.streams }} + {{- range $k, $v := .Values.global.events.streams }} - name: RS_REDIS_QUEUE_{{ $k }} value: {{ $v }} - {{ - end }} + {{- end }} - name: RS_SOLR_URL value: "http://{{ template "solr.fullname" . }}:{{ .Values.global.solr.port }}/solr" - name: RS_SOLR_CORE From 7beae0bd0a044c8c372950cef099f98eaef8636a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 5 Mar 2024 13:15:05 +0100 Subject: [PATCH 31/61] refactor: solr admin pass to be read from secret --- .../renku/templates/search/search-api-deployment.yaml | 7 +++++-- .../templates/search/search-provision-deployment.yaml | 7 +++++-- helm-chart/renku/values.yaml | 1 - 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/helm-chart/renku/templates/search/search-api-deployment.yaml b/helm-chart/renku/templates/search/search-api-deployment.yaml index 0a7b571410..106e1d2e35 100644 --- a/helm-chart/renku/templates/search/search-api-deployment.yaml +++ b/helm-chart/renku/templates/search/search-api-deployment.yaml @@ -36,9 +36,12 @@ spec: - name: RS_SOLR_CORE value: {{ first .Values.solr.coreNames }} - name: RS_SOLR_USER - value: {{ .Values.solr.auth.adminUsername | quote }} + value: {{ default "admin" .Values.solr.auth.adminUsername | quote }} - name: RS_SOLR_PASS - value: {{ .Values.solr.auth.adminPassword | quote }} + valueFrom: + secretKeyRef: + name: {{ template "solr.fullname" . }} + key: solr-password - name: RS_SOLR_DEFAULT_COMMIT_WITHIN value: "500ms" - name: RS_SOLR_LOG_MESSAGE_BODIES diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml index bae9b89b13..e94b97398a 100644 --- a/helm-chart/renku/templates/search/search-provision-deployment.yaml +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -57,9 +57,12 @@ spec: - name: RS_SOLR_CORE value: {{ first .Values.solr.coreNames }} - name: RS_SOLR_USER - value: {{ .Values.solr.auth.adminUsername | quote }} + value: {{ default "admin" .Values.solr.auth.adminUsername | quote }} - name: RS_SOLR_PASS - value: {{ .Values.solr.auth.adminPassword | quote }} + valueFrom: + secretKeyRef: + name: {{ template "solr.fullname" . }} + key: solr-password - name: RS_SOLR_DEFAULT_COMMIT_WITHIN value: "500ms" - name: RS_SOLR_LOG_MESSAGE_BODIES diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 79a4fa3d8c..2cef9efd69 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -488,7 +488,6 @@ solr: replicaCount: 1 auth: adminUsername: renku - adminPassword: renku coreNames: - renku-search javaMem: "-Xmx512M" From 4de494e135ad18207a4b4150a4736b868ae7e39c Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 5 Mar 2024 15:41:48 +0100 Subject: [PATCH 32/61] add gateway config for search endpoint --- helm-chart/renku/templates/gateway/deployment-revproxy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm-chart/renku/templates/gateway/deployment-revproxy.yaml b/helm-chart/renku/templates/gateway/deployment-revproxy.yaml index 7ea68a8a1f..74bbe41e71 100644 --- a/helm-chart/renku/templates/gateway/deployment-revproxy.yaml +++ b/helm-chart/renku/templates/gateway/deployment-revproxy.yaml @@ -69,6 +69,8 @@ spec: value: {{ printf "http://%s-gateway-auth" .Release.Name }} - name: REVPROXY_RENKU_SERVICES_DATA_SERVICE value: {{ printf "http://%s-data-service" .Release.Name | quote }} + - name: REVPROXY_RENKU_SERVICES_SEARCH + value: {{ template "renku.search.searchApi.fullname" . }} - name: REVPROXY_RENKU_SERVICES_KEYCLOAK value: {{ include "renku.keycloakUrl" . | quote }} - name: REVPROXY_PORT From 3b4daaadf8a1bc66ae38af9fcfc3172d3164c820 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 5 Mar 2024 16:11:28 +0100 Subject: [PATCH 33/61] fix gateway search url --- helm-chart/renku/templates/gateway/deployment-revproxy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/templates/gateway/deployment-revproxy.yaml b/helm-chart/renku/templates/gateway/deployment-revproxy.yaml index 74bbe41e71..1cf3cc3419 100644 --- a/helm-chart/renku/templates/gateway/deployment-revproxy.yaml +++ b/helm-chart/renku/templates/gateway/deployment-revproxy.yaml @@ -70,7 +70,7 @@ spec: - name: REVPROXY_RENKU_SERVICES_DATA_SERVICE value: {{ printf "http://%s-data-service" .Release.Name | quote }} - name: REVPROXY_RENKU_SERVICES_SEARCH - value: {{ template "renku.search.searchApi.fullname" . }} + value: {{ printf "http://%s-search-api" .Release.Name | quote }} - name: REVPROXY_RENKU_SERVICES_KEYCLOAK value: {{ include "renku.keycloakUrl" . | quote }} - name: REVPROXY_PORT From 12ed0f825bfdfce3698c79a6b363362d3ffaeda6 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 6 Mar 2024 10:59:45 +0100 Subject: [PATCH 34/61] refactor: solr admin username removed from the values.yaml --- helm-chart/renku/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 2fba7643db..164badcb72 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -486,8 +486,6 @@ solr: cloudEnabled: false collectionReplicas: 0 replicaCount: 1 - auth: - adminUsername: renku coreNames: - renku-search javaMem: "-Xmx512M" From a063aabeebd39a022c5c065b5eaefcc97b37b699 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 6 Mar 2024 11:00:13 +0100 Subject: [PATCH 35/61] chore: search services upgraded to 0.0.25 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 164badcb72..f57d39ef59 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1334,7 +1334,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.24" + tag: "0.0.25" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1347,7 +1347,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.24" + tag: "0.0.25" pullPolicy: IfNotPresent service: type: ClusterIP From 7d30837445c212e2673eefbe417179bf28e26d58 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 6 Mar 2024 11:37:04 +0100 Subject: [PATCH 36/61] chore: ingress for /apiv2/search taken out --- helm-chart/renku/templates/ingress.yaml | 7 ------- helm-chart/renku/templates/swagger.yaml | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/helm-chart/renku/templates/ingress.yaml b/helm-chart/renku/templates/ingress.yaml index fb9c9064c0..f19b563b91 100644 --- a/helm-chart/renku/templates/ingress.yaml +++ b/helm-chart/renku/templates/ingress.yaml @@ -127,13 +127,6 @@ spec: port: number: 80 {{- end }} - - path: /apiv2/search - pathType: Prefix - backend: - service: - name: {{ $searchApiFullname }} - port: - number: 80 {{- if $swaggerEnabled }} - path: /swagger pathType: Prefix diff --git a/helm-chart/renku/templates/swagger.yaml b/helm-chart/renku/templates/swagger.yaml index 38293f5cdc..17446ae4a8 100644 --- a/helm-chart/renku/templates/swagger.yaml +++ b/helm-chart/renku/templates/swagger.yaml @@ -30,7 +30,7 @@ spec: {"url": "/api/kg/spec.json", "name": "knowledge graph"}, {"url": "/api/notebooks/spec.json", "name": "notebooks service"}, {"url": "/api/data/spec.json", "name": "data service"}, - {"url": "/apiv2/search/spec.json", "name": "search service"} + {"url": "/api/search/spec.json", "name": "search service"} ] - name: OAUTH2_REDIRECT_URL value: https://{{ .Values.global.renku.domain }}/swagger/oauth2-redirect.html From f45f2f0e18a7bc40599894e8c04153045788445f Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 6 Mar 2024 11:53:36 +0100 Subject: [PATCH 37/61] chore: search services upgraded to 0.0.26 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index f57d39ef59..a8c440ad0a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1334,7 +1334,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.25" + tag: "0.0.26" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1347,7 +1347,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.25" + tag: "0.0.26" pullPolicy: IfNotPresent service: type: ClusterIP From 8b4ced2cfa893a06d5e9da8683828241c0684179 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 6 Mar 2024 12:03:29 +0100 Subject: [PATCH 38/61] chore: search services upgraded to 0.0.27 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index a8c440ad0a..260554cf47 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1334,7 +1334,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.26" + tag: "0.0.27" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1347,7 +1347,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.26" + tag: "0.0.27" pullPolicy: IfNotPresent service: type: ClusterIP From 9fae5cd084b7428626964d645d4e0a669c7fabb4 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Thu, 7 Mar 2024 10:43:32 +0100 Subject: [PATCH 39/61] chore: search services upgraded to 0.0.28 - user entity --- helm-chart/renku/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 260554cf47..60a3988d4a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -83,6 +83,7 @@ global: events: streams: projectCreated: "project.created" + userAdded: "user.added" ## Note that the graph will not turned on by default until renku 0.4.0 graph: dbEventLog: @@ -1334,7 +1335,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.27" + tag: "0.0.28" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1347,7 +1348,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.27" + tag: "0.0.28" pullPolicy: IfNotPresent service: type: ClusterIP From 3201f60664dcccb747f76a6cc2d429774cbe5dac Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Thu, 7 Mar 2024 12:03:26 +0100 Subject: [PATCH 40/61] chore: search services upgraded to 0.0.29 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 60a3988d4a..98142f66f6 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1335,7 +1335,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.28" + tag: "0.0.29" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1348,7 +1348,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.28" + tag: "0.0.29" pullPolicy: IfNotPresent service: type: ClusterIP From 3d87a5a52e857ae6a6d1ddd952ada5e1d22be6b4 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 11 Mar 2024 10:52:57 +0100 Subject: [PATCH 41/61] chore: search services upgraded to 0.0.30 - support for the update events & bugfixes --- helm-chart/renku/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 98142f66f6..24f84ca0ed 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -83,7 +83,9 @@ global: events: streams: projectCreated: "project.created" + projectUpdated: "project.updated" userAdded: "user.added" + userUpdated: "user.updated" ## Note that the graph will not turned on by default until renku 0.4.0 graph: dbEventLog: @@ -1335,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.29" + tag: "0.0.30" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1348,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.29" + tag: "0.0.30" pullPolicy: IfNotPresent service: type: ClusterIP From aa099d7b78ed782b40e84c66ec7fb0e43e7db53f Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 13 Mar 2024 13:49:58 +0100 Subject: [PATCH 42/61] chore: search services upgraded to 0.0.30 - support for the auth and project removal events --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 24f84ca0ed..3ee22fbc6f 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1337,7 +1337,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.30" + tag: "0.0.31" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1350,7 +1350,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.30" + tag: "0.0.31" pullPolicy: IfNotPresent service: type: ClusterIP From 405a936b3fd6699ea1160247a3fed57208a5a93c Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 13 Mar 2024 14:02:01 +0100 Subject: [PATCH 43/61] chore: stream definitions in values.yaml --- helm-chart/renku/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 3ee22fbc6f..57cd274cca 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -84,8 +84,13 @@ global: streams: projectCreated: "project.created" projectUpdated: "project.updated" + projectRemoved: "project.removed" + projectAuthAdded: "projectAuth.added" + projectAuthUpdated: "projectAuth.updated" + projectAuthRemoved: "projectAuth.removed" userAdded: "user.added" userUpdated: "user.updated" + userRemoved: "user.removed" ## Note that the graph will not turned on by default until renku 0.4.0 graph: dbEventLog: From 95f8067d08431dfa6c19a4cee608dee872c96a2a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 13 Mar 2024 14:15:57 +0100 Subject: [PATCH 44/61] chore: search services upgraded to 0.0.32 - streams names aligning --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 57cd274cca..88f23086a7 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.31" + tag: "0.0.32" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.31" + tag: "0.0.32" pullPolicy: IfNotPresent service: type: ClusterIP From 32106019c73789e1b9e29df20cae20594e375545 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 14 Mar 2024 08:37:26 +0100 Subject: [PATCH 45/61] chore: Update search services --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 88f23086a7..4cebef02ba 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.32" + tag: "22887c3d76ce" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.32" + tag: "22887c3d76ce" pullPolicy: IfNotPresent service: type: ClusterIP From 0ba265692735f2d533b6a64f9714dad2bb57dc0f Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 14 Mar 2024 13:36:53 +0100 Subject: [PATCH 46/61] add redis values forusers sync svc --- .../templates/keycloak-users-sync-cronjob.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml index c109f4875a..ba8c616392 100644 --- a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml +++ b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml @@ -48,6 +48,21 @@ spec: key: dataServiceKeycloakClientSecret - name: TOTAL_USER_SYNC value: "false" + - name: REDIS_HOST + value: {{ .Values.global.redis.host | quote }} + - name: REDIS_PORT + value: {{ .Values.global.redis.port | quote }} + - name: REDIS_DATABASE + value: {{ .Values.global.redis.dbIndex.events| quote }} + - name: REDIS_IS_SENTINEL + value: {{ .Values.global.redis.sentinel.enabled | quote }} + - name: REDIS_MASTER_SET + value: {{ .Values.global.redis.sentinel.masterSet | quote }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: {{ .Values.global.redis.existingSecretPasswordKey }} {{- include "certificates.env.python" . | nindent 16 }} volumeMounts: {{- include "certificates.volumeMounts.system" . | nindent 16 }} From b552c615c754d2bf6ab411a4ff6acd07c93d1521 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 14 Mar 2024 14:45:33 +0100 Subject: [PATCH 47/61] add redis access to user sync job --- .../templates/keycloak-users-sync-cronjob.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml index ba8c616392..4447ff80e9 100644 --- a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml +++ b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml @@ -7,6 +7,7 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} chart: {{ template "renku.chart" . }} + {{ .Values.global.redis.clientLabel | toYaml | nindent 4 }} spec: schedule: "*/2 * * * *" concurrencyPolicy: Forbid @@ -79,6 +80,7 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} chart: {{ template "renku.chart" . }} + {{ .Values.global.redis.clientLabel | toYaml | nindent 4 }} spec: schedule: "0 3 * * *" concurrencyPolicy: Forbid @@ -120,6 +122,21 @@ spec: key: dataServiceKeycloakClientSecret - name: TOTAL_USER_SYNC value: "true" + - name: REDIS_HOST + value: {{ .Values.global.redis.host | quote }} + - name: REDIS_PORT + value: {{ .Values.global.redis.port | quote }} + - name: REDIS_DATABASE + value: {{ .Values.global.redis.dbIndex.events| quote }} + - name: REDIS_IS_SENTINEL + value: {{ .Values.global.redis.sentinel.enabled | quote }} + - name: REDIS_MASTER_SET + value: {{ .Values.global.redis.sentinel.masterSet | quote }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: {{ .Values.global.redis.existingSecretPasswordKey }} {{- include "certificates.env.python" . | nindent 16 }} volumeMounts: {{- include "certificates.volumeMounts.system" . | nindent 16 }} From 827237f5e110e8bfed3cf3053826fe5058cb2069 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 14 Mar 2024 15:10:58 +0100 Subject: [PATCH 48/61] move annotation to correct place --- helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml index 4447ff80e9..a8cfbac3b8 100644 --- a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml +++ b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml @@ -7,7 +7,6 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} chart: {{ template "renku.chart" . }} - {{ .Values.global.redis.clientLabel | toYaml | nindent 4 }} spec: schedule: "*/2 * * * *" concurrencyPolicy: Forbid @@ -19,6 +18,7 @@ spec: metadata: labels: app: keycloak-sync + {{ .Values.global.redis.clientLabel | toYaml | nindent 12 }} spec: initContainers: {{- include "certificates.initContainer" . | nindent 12 }} @@ -92,6 +92,7 @@ spec: metadata: labels: app: keycloak-sync + {{ .Values.global.redis.clientLabel | toYaml | nindent 12 }} spec: initContainers: {{- include "certificates.initContainer" . | nindent 12 }} From eed023ac3020c19da746909d5f5d37f679453bad Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 15 Mar 2024 14:43:19 +0100 Subject: [PATCH 49/61] chore: update search services to 0.0.33 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 4cebef02ba..9946887626 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "22887c3d76ce" + tag: "0.0.33" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "22887c3d76ce" + tag: "0.0.33" pullPolicy: IfNotPresent service: type: ClusterIP From 9e920a4763e31999f021c86a804ae3ea10719bf2 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 19 Mar 2024 14:59:20 +0100 Subject: [PATCH 50/61] chore: search services upgraded to 0.0.34 - role filtering; prometheus metrics --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 9946887626..7e4d8629e1 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.33" + tag: "0.0.34" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.33" + tag: "0.0.34" pullPolicy: IfNotPresent service: type: ClusterIP From 00ae701657b1d45be1a1dcff170317a0fde48508 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 19 Mar 2024 15:10:00 +0100 Subject: [PATCH 51/61] feat: search services to expose metrics API --- .../templates/search/search-api-service.yaml | 4 ++++ .../search/search-provision-deployment.yaml | 4 ++++ .../search/search-provision-service.yaml | 23 +++++++++++++++++++ helm-chart/renku/values.yaml | 1 + 4 files changed, 32 insertions(+) create mode 100644 helm-chart/renku/templates/search/search-provision-service.yaml diff --git a/helm-chart/renku/templates/search/search-api-service.yaml b/helm-chart/renku/templates/search/search-api-service.yaml index 25c832e3c2..0bff5aca6b 100644 --- a/helm-chart/renku/templates/search/search-api-service.yaml +++ b/helm-chart/renku/templates/search/search-api-service.yaml @@ -7,6 +7,10 @@ metadata: chart: {{ template "renku.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '8080' spec: type: {{ .Values.search.searchApi.service.type }} ports: diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml index e94b97398a..21f122af3f 100644 --- a/helm-chart/renku/templates/search/search-provision-deployment.yaml +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -69,6 +69,10 @@ spec: value: "false" - name: JAVA_OPTS value: "-Xmx{{ .Values.search.searchProvision.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational" + ports: + - name: http-search-prov + containerPort: 8081 + protocol: TCP resources: {{- toYaml .Values.search.searchProvision.resources | nindent 12 }} volumeMounts: diff --git a/helm-chart/renku/templates/search/search-provision-service.yaml b/helm-chart/renku/templates/search/search-provision-service.yaml new file mode 100644 index 0000000000..ea7b54420d --- /dev/null +++ b/helm-chart/renku/templates/search/search-provision-service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "renku.search.searchProvision.fullname" . }} + labels: + app: {{ template "renku.search.searchProvision.name" . }} + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '8081' +spec: + type: {{ .Values.search.searchProvision.service.type }} + ports: + - port: {{ .Values.search.searchProvision.service.port }} + targetPort: http-search-prov + protocol: TCP + name: http + selector: + app: {{ template "renku.search.searchProvision.name" . }} + release: {{ .Release.Name }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 7e4d8629e1..554629a265 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1359,6 +1359,7 @@ search: pullPolicy: IfNotPresent service: type: ClusterIP + port: 8081 resources: requests: memory: 256Mi From 1bf8dbef691103ca0df881830ad532a98aef5c12 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 19 Mar 2024 15:15:19 +0100 Subject: [PATCH 52/61] fix: search-provision port definition too long --- .../renku/templates/search/search-provision-deployment.yaml | 2 +- helm-chart/renku/templates/search/search-provision-service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/templates/search/search-provision-deployment.yaml b/helm-chart/renku/templates/search/search-provision-deployment.yaml index 21f122af3f..ce58800028 100644 --- a/helm-chart/renku/templates/search/search-provision-deployment.yaml +++ b/helm-chart/renku/templates/search/search-provision-deployment.yaml @@ -70,7 +70,7 @@ spec: - name: JAVA_OPTS value: "-Xmx{{ .Values.search.searchProvision.jvmXmx }} -XX:+UseZGC -XX:+ZGenerational" ports: - - name: http-search-prov + - name: http-searchprov containerPort: 8081 protocol: TCP resources: diff --git a/helm-chart/renku/templates/search/search-provision-service.yaml b/helm-chart/renku/templates/search/search-provision-service.yaml index ea7b54420d..b1aa6d4687 100644 --- a/helm-chart/renku/templates/search/search-provision-service.yaml +++ b/helm-chart/renku/templates/search/search-provision-service.yaml @@ -15,7 +15,7 @@ spec: type: {{ .Values.search.searchProvision.service.type }} ports: - port: {{ .Values.search.searchProvision.service.port }} - targetPort: http-search-prov + targetPort: http-searchprov protocol: TCP name: http selector: From 68f3e555f5ac142552a8eeecca93778c0efefb4d Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 20 Mar 2024 10:01:57 +0100 Subject: [PATCH 53/61] chore: search services upgraded to 0.0.35 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 554629a265..ea308355b5 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.34" + tag: "0.0.35" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.34" + tag: "0.0.35" pullPolicy: IfNotPresent service: type: ClusterIP From 28b74269c1a236f81945376d203dab4df26d1025 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Fri, 22 Mar 2024 10:41:31 +0100 Subject: [PATCH 54/61] chore: search services upgraded to 0.0.36 - partial docs --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index ea308355b5..53b53f389b 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1342,7 +1342,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.35" + tag: "0.0.36" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1355,7 +1355,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.35" + tag: "0.0.36" pullPolicy: IfNotPresent service: type: ClusterIP From 9f5ac79293d619b404f7adc5152f6a8ec4c20ff7 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 25 Mar 2024 14:45:39 +0100 Subject: [PATCH 55/61] update dependendies and changelog --- CHANGELOG.rst | 14 ++++++++++++++ helm-chart/renku/values.yaml | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f87d60ce6e..77074e7d4d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,20 @@ 0.50.x ------ +This release introduces new search functionality for Renku 2.0 + + +Internal Changes +~~~~~~~~~~~~~~~~ + +**New Features** + +- **Data services**: Initial support for project and user search for Renku 2.0 (alpha release) + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-data-services 0.6.0 `_ 0.49.0 diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 21137389d1..ab4d02333f 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1547,12 +1547,12 @@ initDb: dataService: image: repository: renku/renku-data-service - tag: "0.5.0" + tag: "0.6.0" pullPolicy: IfNotPresent keycloakSync: image: repository: renku/keycloak-sync - tag: "0.5.0" + tag: "0.6.0" pullPolicy: IfNotPresent service: type: ClusterIP From c4b7c7322e3e2b7cdd98e3be48d37adaf11fbac8 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 25 Mar 2024 15:15:14 +0100 Subject: [PATCH 56/61] update changelog and bump gateway --- CHANGELOG.rst | 2 ++ helm-chart/renku/values.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 77074e7d4d..5f48c72478 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-data-services 0.6.0 `_ +- `renku-gateway 0.24.0 `_ + 0.49.0 diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index ab4d02333f..00da97a856 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1241,7 +1241,7 @@ gateway: ## Define the image for the auth middleware auth: repository: renku/renku-gateway - tag: "0.23.1" + tag: "0.24.0" pullPolicy: IfNotPresent service: type: ClusterIP From 5ed8427433aef82adfbf8ffc0a53cf6549b6df55 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 26 Mar 2024 14:00:27 +0100 Subject: [PATCH 57/61] chore: search services upgraded to 0.0.37 --- helm-chart/renku/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 00da97a856..d87e2af0ea 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1340,7 +1340,7 @@ search: replicas: 1 image: repository: renku/search-api - tag: "0.0.36" + tag: "0.0.37" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1353,7 +1353,7 @@ search: replicas: 1 image: repository: renku/search-provision - tag: "0.0.36" + tag: "0.0.37" pullPolicy: IfNotPresent service: type: ClusterIP From 4df363493fa14a79ec890673892da7bac49a1185 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 26 Mar 2024 15:33:45 +0100 Subject: [PATCH 58/61] bump gateway rev proxy --- helm-chart/renku/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index d87e2af0ea..4d3c794fd9 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1277,7 +1277,7 @@ gateway: reverseProxy: image: repository: renku/renku-revproxy - tag: "0.23.0" + tag: "0.24.0" pullPolicy: IfNotPresent metrics: enabled: true From cf7c0c300d13b189d157a6453c314037acb8f36a Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 26 Mar 2024 17:44:07 +0100 Subject: [PATCH 59/61] bump renku-ui, renku-notebooks (#3553) --- helm-chart/renku/values.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 4d3c794fd9..eeba5ee10e 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -650,7 +650,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.20.1" + tag: "3.21.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -809,7 +809,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.20.1" + tag: "3.21.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" @@ -954,7 +954,7 @@ notebooks: targetCPUUtilizationPercentage: 50 image: repository: renku/renku-notebooks - tag: "1.21.0" + tag: "1.22.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -1072,15 +1072,15 @@ notebooks: gitRpcServer: image: name: renku/git-rpc-server - tag: "1.21.0" + tag: "1.22.0" gitHttpsProxy: image: name: renku/git-https-proxy - tag: "1.21.0" + tag: "1.22.0" gitClone: image: name: renku/git-clone - tag: "1.21.0" + tag: "1.22.0" service: type: ClusterIP port: 80 @@ -1133,12 +1133,12 @@ notebooks: sessionTypes: ["registered"] image: repository: renku/renku-notebooks-tests - tag: "1.21.0" + tag: "1.22.0" pullPolicy: IfNotPresent k8sWatcher: image: repository: renku/k8s-watcher - tag: "1.21.0" + tag: "1.22.0" pullPolicy: IfNotPresent resources: {} replicaCount: 1 @@ -1151,7 +1151,7 @@ notebooks: enabled: false image: repository: renku/ssh-jump-host - tag: "1.21.0" + tag: "1.22.0" pullPolicy: IfNotPresent resources: {} replicaCount: 1 From 2c485c9ac8c3873bb305c61f8b99b72929e2cf31 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 26 Mar 2024 19:01:43 +0100 Subject: [PATCH 60/61] chore: changelog update --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5f48c72478..f41da390c3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,12 +12,14 @@ Internal Changes **New Features** - **Data services**: Initial support for project and user search for Renku 2.0 (alpha release) +- **Search services**: Initial support for project and user search for Renku 2.0 (alpha release) Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-data-services 0.6.0 `_ - `renku-gateway 0.24.0 `_ +- `renku-search 0.0.37 `_ From f9281204f756eb2ac08816c18dcf3d518c782875 Mon Sep 17 00:00:00 2001 From: Alessandro Degano <40891147+aledegano@users.noreply.github.com> Date: Wed, 27 Mar 2024 09:17:36 +0100 Subject: [PATCH 61/61] feat: enable Network Policy for Solr. (#3512) This change to the values files of Solr enables a network policy to Solr such that only `search-api` and `search-provision` are allowed to connect to it. --- helm-chart/renku/values.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index eeba5ee10e..48453e5b74 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -498,7 +498,17 @@ solr: - renku-search javaMem: "-Xmx512M" networkPolicy: - enabled: false # Awaiting instructions from: https://github.com/bitnami/charts/issues/23424 + allowExternal: false + extraIngress: + - ports: + - port: 8983 + from: + - podSelector: + matchLabels: + app: search-api + - podSelector: + matchLabels: + app: search-provision persistence: enabled: true size: 8Gi