feat: renku native projects (#3376)

Co-authored-by: Laura <[email protected]> Co-authored-by: M. Alisafaee <[email protected]> Co-authored-by: Jakub Chrobasik <[email protected]> Co-authored-by: Ralf Grubenmann <[email protected]>
SwissDataScienceCenter · Mar 5, 2024 · d1f252b · d1f252b
1 parent ed21d78
commit d1f252b
Show file tree

Hide file tree

Showing 13 changed files with 374 additions and 50 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,7 +3,11 @@
 0.49.0
 ------
 
-This release contains bug fixes to renku core service related to project migration.
+The release contains bug fixes to renku core service related to project migration.
+
+This release also contains initial support for next generation 'Renku 1.0' functionality. However,
+Renku 1.0 is still in early development and is not yet accessible to users. For more information,
+see our [roadmap](https://github.com/SwissDataScienceCenter/renku-design-docs/blob/main/roadmap.md).
 
 **Note for administrators**: this release includes breaking changes due to upgrading PostgreSQL to 16.1.0.
 This requires modifying the values file to work with the new PostgreSQL Helm chart.
@@ -20,17 +24,24 @@ User-Facing Changes
 Internal Changes
 ~~~~~~~~~~~~~~~~
 
+**New Features**
+
+- **Data services**: Initial support for Renku 1.0 projects (alpha release)
+
 **Improvements**
 
 - **csi-rclone**: added rclone logs to regular node-plugin logs.
   (`#11 <https://github.com/SwissDataScienceCenter/csi-rclone/pull/11>`_).
 
+
 Individual Components
 ~~~~~~~~~~~~~~~~~~~~~
 
 - `renku-python 2.9.2 <https://github.com/SwissDataScienceCenter/renku-python/releases/tag/2.9.2>`_
+- `renku-data-services 0.5.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.5.0>`_
 - `csi-rclone 0.1.7 <https://github.com/SwissDataScienceCenter/csi-rclone/releases/tag/v0.1.7>`_
 
+
 0.48.1
 ------
 

diff --git a/acceptance-tests/README.md b/acceptance-tests/README.md
@@ -204,7 +204,6 @@ The test are built using the Page Object Pattern (e.g.
 https://www.pluralsight.com/guides/getting-started-with-page-object-pattern-for-your-selenium-tests) which in
 short is about wrapping an UI page into a class/object and using it in the test script.
 
-
 As mentioned above there's a `target/tests-execution.log` file where tests debug statements from tests execution are written.
 
 ## Project organization

diff --git a/chartpress.yaml b/chartpress.yaml
@@ -19,6 +19,7 @@ charts:
       - helm-chart
       - acceptance-tests
       - scripts/init-realm
+      - scripts/init-db
     images:
       tests:
         buildArgs:

diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl
@@ -164,3 +164,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- define "renku.keycloak.realm" -}}
 {{ .Values.global.keycloak.realm | default "Renku" }}
 {{- end -}}
+
+{{- define "renku.dataService.keycloak.clientId" -}}
+data-service
+{{- end -}}
diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml
@@ -58,6 +58,13 @@ spec:
               value: {{ (printf "%s://%s/auth/" (include "renku.http" .) .Values.global.renku.domain) | quote }}
             - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS
               value: "RS256"
+            - name: KEYCLOAK_CLIENT_ID
+              value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+            - name: KEYCLOAK_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "renku.fullname" . }}
+                  key: dataServiceKeycloakClientSecret
             - name: SERVER_DEFAULTS
               value: /etc/renku-data-service/server_options/server_defaults.json
             - name: SERVER_OPTIONS

diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml
@@ -0,0 +1,113 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "renku.fullname" . }}-keycloak-sync-events
+  labels:
+    app: keycloak-sync
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ template "renku.chart" . }}
+spec:
+  schedule: "*/2 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:          
+            app: keycloak-sync
+        spec:
+          initContainers:
+            {{- include "certificates.initContainer" . | nindent 12 }}
+          containers:
+            - name: keycloak-sync
+              image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}"
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: DB_HOST
+                  value: {{ template "postgresql.fullname" . }}
+                - name: DB_USER
+                  value: {{ .Values.global.db.common.username }}
+                - name: DB_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.db.common.passwordSecretName }}
+                      key: password
+                - name: KEYCLOAK_URL
+                  value: {{ include "renku.keycloakUrl" . | quote}}
+                - name: KEYCLOAK_REALM
+                  value: {{ include "renku.keycloak.realm" . | quote}}
+                - name: KEYCLOAK_CLIENT_ID
+                  value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+                - name: KEYCLOAK_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ include "renku.fullname" . }}
+                      key: dataServiceKeycloakClientSecret
+                - name: TOTAL_USER_SYNC
+                  value: "false"
+                {{- include "certificates.env.python" . | nindent 16 }}
+              volumeMounts:
+                {{- include "certificates.volumeMounts.system" . | nindent 16 }}
+          restartPolicy: Never
+          volumes:
+            {{- include "certificates.volumes" . | nindent 12 }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "renku.fullname" . }}-keycloak-sync-total
+  labels:
+    app: keycloak-sync
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ template "renku.chart" . }}
+spec:
+  schedule: "0 3 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:          
+            app: keycloak-sync
+        spec:
+          initContainers:
+            {{- include "certificates.initContainer" . | nindent 12 }}
+          containers:
+            - name: keycloak-sync
+              image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}"
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: DB_HOST
+                  value: {{ template "postgresql.fullname" . }}
+                - name: DB_USER
+                  value: {{ .Values.global.db.common.username }}
+                - name: DB_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.global.db.common.passwordSecretName }}
+                      key: password
+                - name: KEYCLOAK_URL
+                  value: {{ include "renku.keycloakUrl" . | quote}}
+                - name: KEYCLOAK_REALM
+                  value: {{ include "renku.keycloak.realm" . | quote}}
+                - name: KEYCLOAK_CLIENT_ID
+                  value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+                - name: KEYCLOAK_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ include "renku.fullname" . }}
+                      key: dataServiceKeycloakClientSecret
+                - name: TOTAL_USER_SYNC
+                  value: "true"
+                {{- include "certificates.env.python" . | nindent 16 }}
+              volumeMounts:
+                {{- include "certificates.volumeMounts.system" . | nindent 16 }}
+          restartPolicy: Never
+          volumes:
+            {{- include "certificates.volumes" . | nindent 12 }}
diff --git a/helm-chart/renku/templates/network-policies.yaml b/helm-chart/renku/templates/network-policies.yaml
@@ -69,6 +69,12 @@ spec:
           namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        - podSelector:
+            matchLabels:
+              app: keycloak-sync
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
       ports:
         - protocol: TCP
           port: 5432

diff --git a/helm-chart/renku/templates/secrets.yaml b/helm-chart/renku/templates/secrets.yaml
@@ -1,4 +1,14 @@
 ---
+{{- $data_service_kc_client_secret := (randAlphaNum 64) | b64enc | quote }}
+
+{{- $renku_secret := lookup "v1" "Secret" .Release.Namespace (include "renku.fullname" .) }}
+{{- if and $renku_secret $renku_secret.data }}
+{{- $data_service_kc_client_secret_test := index $renku_secret.data "dataServiceKeycloakClientSecret" }}
+{{- if $data_service_kc_client_secret_test }}
+{{- $data_service_kc_client_secret = $data_service_kc_client_secret_test }}
+{{- end -}}
+{{- end -}}
+
 apiVersion: v1
 kind: Secret
 metadata:
@@ -8,11 +18,16 @@ metadata:
     chart: {{ template "renku.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
+  annotations:
+    # If "keep" resource policy is removed the secret is deleted post upgrade see https://github.com/helm/helm/issues/8420
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install,pre-upgrade,pre-rollback"
 type: Opaque
 data:
   {{- if .Values.tests.users_json }}
   users.json: {{ .Values.tests.users_json | toJson | b64enc | quote }}
   {{- end }}
+  dataServiceKeycloakClientSecret: {{ $data_service_kc_client_secret }}
 
 
 {{- if and (eq .Values.redis.install true) (eq .Values.redis.createSecret true) }}

diff --git a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml
@@ -86,6 +86,8 @@ spec:
                   key: oidcClientSecret
             - name: RENKU_KC_CLIENT_PUBLIC
               value: "false"
+            - name: RENKU_KC_CLIENT_OAUTH_FLOW
+              value: "authorization_code"
             - name: CLI_KC_CLIENT_ID
               value: renku-cli
             - name: CLI_KC_CLIENT_SECRET
@@ -97,6 +99,8 @@ spec:
               value: "true"
             - name: CLI_KC_CLIENT_ATTRIBUTES
               value: '{"access.token.lifespan": "86400", "oauth2.device.authorization.grant.enabled": true, "oauth2.device.polling.interval": "5"}'
+            - name: CLI_KC_CLIENT_OAUTH_FLOW
+              value: "device"
             - name: UI_KC_CLIENT_ID
               value: "renku-ui"
             - name: UI_KC_CLIENT_SECRET
@@ -106,6 +110,8 @@ spec:
                   key: uiserverClientSecret
             - name: UI_KC_CLIENT_PUBLIC
               value: "false"
+            - name: UI_KC_CLIENT_OAUTH_FLOW
+              value: "authorization_code"
             - name: NOTEBOOKS_KC_CLIENT_ID
               value: {{ .Values.notebooks.oidc.clientId | default "renku-jupyterserver" | quote }}
             - name: NOTEBOOKS_KC_CLIENT_SECRET
@@ -115,12 +121,29 @@ spec:
                   key: notebooksClientSecret
             - name: NOTEBOOKS_KC_CLIENT_PUBLIC
               value: "false"
+            - name: NOTEBOOKS_KC_CLIENT_OAUTH_FLOW
+              value: "authorization_code"
             - name: SWAGGER_KC_CLIENT_ID
               value: swagger
             - name: SWAGGER_KC_CLIENT_PUBLIC
               value: "true"
+            - name: SWAGGER_KC_CLIENT_OAUTH_FLOW
+              value: "authorization_code"
             - name: SWAGGER_KC_CLIENT_ATTRIBUTES
               value: '{"pkce.code.challenge.method": "S256"}'
+            - name: DATASERVICE_KC_CLIENT_ID
+              value: {{ include "renku.dataService.keycloak.clientId" . | quote }}
+            - name: DATASERVICE_KC_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "renku.fullname" . }}
+                  key: dataServiceKeycloakClientSecret
+            - name: DATASERVICE_KC_CLIENT_PUBLIC
+              value: "false"
+            - name: DATASERVICE_KC_CLIENT_OAUTH_FLOW
+              value: "client_credentials"
+            - name: "DATASERVICE_KC_CLIENT_SERVICE_ACCOUNT_ROLES"
+              value: '["view-users", "query-users", "view-events"]'
             - name: PYTHONUNBUFFERED
               value: "0"
             {{- include "certificates.env.python" . | nindent 12 }}

diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml
@@ -1488,8 +1488,13 @@ initDb:
 dataService:
   image:
     repository: renku/renku-data-service
-    tag: "0.4.0"
+    tag: "0.5.0"
     pullPolicy: IfNotPresent
+  keycloakSync:
+    image:
+      repository: renku/keycloak-sync
+      tag: "0.5.0"
+      pullPolicy: IfNotPresent
   service:
     type: ClusterIP
     port: 80

diff --git a/scripts/init-realm/Dockerfile b/scripts/init-realm/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 
 COPY requirements.txt init-realm.py utils.py /app/
 WORKDIR /app