Merge branch 'release-0.52.x' into build/secrets-acceptance-test

SwissDataScienceCenter · May 13, 2024 · eb7a41e · eb7a41e
2 parents 4c61f51 + 2231170
commit eb7a41e
Show file tree

Hide file tree

Showing 13 changed files with 157 additions and 13 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,19 +3,33 @@
 0.52.0
 ------
 
-Renku ``0.52.0`` contains a new secrets-storage service that allows users to store and use secrets in 
-sessions.
+Renku ``0.52.0`` introduces a new secrets-storage service that allows users to store and use secrets in sessions.
 
+This release also contains new features related to Renku 2.0. However, Renku 2.0 is still
+in early development and is not yet accessible to users. For more information, see our
+[blog](https://blog.renkulab.io/renku-2).
+
+User-Facing Changes
+~~~~~~~~~~~~~~~~~~~
 
 **🌟 New Features**
 
 - **Data Services**: Added new secrets storage service for managing user session secrets, including
   new endpoints on data-service to manage these secrets.
 
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**🌟 New Features**
+
+- **Data Services**: Updated the authorization for access to Renku 2.0 projects to work with Authzed DB,
+  a 3rd party database dedicated to saving authorization data and making authorization decisions
+
 Individual Components
 ~~~~~~~~~~~~~~~~~~~~~
 
 - `renku-data-services 0.9.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.9.0>`_
+- `renku-data-services 0.10.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.10.0>`_
 - `renku-notebooks 1.23.0 <https://github.com/SwissDataScienceCenter/renku-notebooks/releases/tag/1.23.0>`_
 
 0.51.1

diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -15,6 +15,7 @@ astroquery
 attrs
 auditability
 auth
+Authzed
 autocomplete
 autosave
 autosaved
@@ -66,6 +67,8 @@ Datashim
 dataverse
 Dataverse
 datetimes
+db
+DB
 deployer
 deserialize
 Deserialize

diff --git a/helm-chart/renku/templates/_certificates-env.tpl b/helm-chart/renku/templates/_certificates-env.tpl
@@ -4,3 +4,8 @@
 - name: SSL_CERT_FILE
   value: /etc/ssl/certs/ca-certificates.crt
 {{- end -}}
+
+{{- define "certificates.env.grpc" -}}
+- name: GRPC_DEFAULT_SSL_ROOTS_FILE_PATH
+  value: /etc/ssl/certs/ca-certificates.crt
+{{- end -}}
diff --git a/helm-chart/renku/templates/_certificates-init-container.tpl b/helm-chart/renku/templates/_certificates-init-container.tpl
@@ -10,9 +10,7 @@
   volumeMounts:
     - name: etc-ssl-certs
       mountPath: /etc/ssl/certs/
-{{- if $customCAsEnabled }}
     - name: custom-ca-certs
       mountPath: /usr/local/share/ca-certificates
       readOnly: true
 {{- end -}}
-{{- end -}}
diff --git a/helm-chart/renku/templates/_certificates-volumes.tpl b/helm-chart/renku/templates/_certificates-volumes.tpl
@@ -3,14 +3,19 @@
 - name: etc-ssl-certs
   emptyDir:
     medium: "Memory"
-{{- if $customCAsEnabled }}
 - name: custom-ca-certs
   projected:
     defaultMode: 0444
     sources:
+      - secret:
+          name: {{ include "renku.CASecretName" . }}
+          items:
+            - key: tls.crt
+              path: {{ include "renku.CASecretName" . }}-internal-communication-ca.crt
+    {{- if $customCAsEnabled }}
     {{- range $customCA := .Values.global.certificates.customCAs }}
       - secret:
           name: {{ $customCA.secret }}
     {{- end -}}
-{{- end -}}
+    {{- end -}}
 {{- end -}}
diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl
@@ -168,3 +168,11 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- define "renku.dataService.keycloak.clientId" -}}
 data-service
 {{- end -}}
+
+{{- define "renku.authz.tlsSecretName" -}}
+renku-authz-tls-cert
+{{- end -}}
+
+{{- define "renku.CASecretName" -}}
+renku-ca
+{{- end -}}
diff --git a/helm-chart/renku/templates/authz/deployment.yaml b/helm-chart/renku/templates/authz/deployment.yaml
@@ -30,6 +30,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "certificates.initContainer" . | nindent 8 }}
         - name: migrate-datastore
           # NOTE: Have to use the -debug image here because the regular one does not have a shell
           # and the database migration will not read the db connection uri string from an env variable
@@ -63,12 +64,28 @@ spec:
                 secretKeyRef:
                   name: {{ template "renku.fullname" . }}-authz
                   key: SPICEDB_DATASTORE_CONN_URI
+            {{- include "certificates.env.python" . | nindent 12 }}
+            {{- include "certificates.env.grpc" . | nindent 12 }}
+          volumeMounts:
+            {{- include "certificates.volumeMounts.system" . | nindent 12 }}
+            - name: internal-tls-authz-cert
+              mountPath: /etc/internal-tls-authz-cert
+              readOnly: true
           args:
             - serve
             - --http-enabled=true
             - --datastore-engine=postgres
             - --grpc-enabled=true
             - --telemetry-endpoint=
+            - --grpc-tls-cert-path=/etc/internal-tls-authz-cert/tls.crt
+            - --grpc-tls-key-path=/etc/internal-tls-authz-cert/tls.key
+            - --http-tls-cert-path=/etc/internal-tls-authz-cert/tls.crt
+            - --http-tls-key-path=/etc/internal-tls-authz-cert/tls.key
+            - --log-format=json
+            - --datastore-conn-pool-write-min-open=3
+            - --datastore-conn-pool-write-max-open=6
+            - --datastore-conn-pool-read-min-open=5
+            - --datastore-conn-pool-read-max-open=10
           ports:
             - name: http
               containerPort: 8443
@@ -84,13 +101,17 @@ spec:
               command:
                 - grpc_health_probe
                 - -v
-                - -addr=localhost:50051
+                - -addr=127.0.0.1:50051
+                - -tls
+                - -tls-server-name={{ template "renku.fullname" . }}-authz
           readinessProbe:
             exec:
               command:
                 - grpc_health_probe
                 - -v
-                - -addr=localhost:50051
+                - -addr=127.0.0.1:50051
+                - -tls
+                - -tls-server-name={{ template "renku.fullname" . }}-authz
           resources:
             {{- toYaml .Values.authz.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -105,3 +126,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      volumes:
+        {{- include "certificates.volumes" . | nindent 8 }}
+        - name: internal-tls-authz-cert
+          secret:
+            secretName: {{ include "renku.authz.tlsSecretName" . }}
diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml
@@ -103,7 +103,17 @@ spec:
               value: {{ .Values.dataService.sentry.environment | quote }}
             - name: SENTRY_SAMPLE_RATE
               value: {{ .Values.dataService.sentry.sampleRate | quote }}
-            {{- include "certificates.env.python" $ | nindent 12 }}
+            - name: AUTHZ_DB_HOST
+              value: {{ include "renku.fullname" . }}-authz
+            - name: AUTHZ_DB_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "renku.fullname" . }}-authz
+                  key: SPICEDB_GRPC_PRESHARED_KEY
+            - name: AUTHZ_DB_GRPC_PORT
+              value: "50051"
+            {{- include "certificates.env.python" . | nindent 12 }}
+            {{- include "certificates.env.grpc" . | nindent 12 }}
           volumeMounts:
             - name: server-options
               mountPath: /etc/renku-data-service/server_options

diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml
@@ -64,7 +64,17 @@ spec:
                     secretKeyRef:
                       name: {{ .Values.global.redis.existingSecret }}
                       key: {{ .Values.global.redis.existingSecretPasswordKey }}
+                - name: AUTHZ_DB_HOST
+                  value: {{ include "renku.fullname" . }}-authz
+                - name: AUTHZ_DB_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ template "renku.fullname" . }}-authz
+                      key: SPICEDB_GRPC_PRESHARED_KEY
+                - name: AUTHZ_DB_GRPC_PORT
+                  value: "50051"
                 {{- include "certificates.env.python" . | nindent 16 }}
+                {{- include "certificates.env.grpc" . | nindent 16 }}
               volumeMounts:
                 {{- include "certificates.volumeMounts.system" . | nindent 16 }}
           restartPolicy: Never
@@ -138,7 +148,17 @@ spec:
                     secretKeyRef:
                       name: {{ .Values.global.redis.existingSecret }}
                       key: {{ .Values.global.redis.existingSecretPasswordKey }}
+                - name: AUTHZ_DB_HOST
+                  value: {{ include "renku.fullname" . }}-authz
+                - name: AUTHZ_DB_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ template "renku.fullname" . }}-authz
+                      key: SPICEDB_GRPC_PRESHARED_KEY
+                - name: AUTHZ_DB_GRPC_PORT
+                  value: "50051"
                 {{- include "certificates.env.python" . | nindent 16 }}
+                {{- include "certificates.env.grpc" . | nindent 16 }}
               volumeMounts:
                 {{- include "certificates.volumeMounts.system" . | nindent 16 }}
           restartPolicy: Never

diff --git a/helm-chart/renku/templates/secret-renku-internal-tls.yaml b/helm-chart/renku/templates/secret-renku-internal-tls.yaml
@@ -0,0 +1,54 @@
+{{- $renkuFullname := include "renku.fullname" . -}}
+{{- $ca := genCA "renku-ca" 3650000 }}
+{{- $authz_svc_name := cat $renkuFullname "-authz" | nospace -}}
+{{- $authz_cert := genSignedCert $authz_svc_name nil (list $authz_svc_name) 3650000 $ca }}
+{{- $ca_crt := $ca.Cert | b64enc | quote }}
+{{- $ca_key := $ca.Key | b64enc | quote }}
+{{- $authz_crt := $authz_cert.Cert | b64enc | quote }}
+{{- $authz_key := $authz_cert.Key | b64enc | quote }}
+
+{{- $secretCAName := include "renku.CASecretName" . }}
+{{- $secretAuthzCertName := include "renku.authz.tlsSecretName" . }}
+{{- $secretCA := (lookup "v1" "Secret" .Release.Namespace $secretCAName) }}
+{{- $secretAuthzCert := (lookup "v1" "Secret" .Release.Namespace $secretAuthzCertName) }}
+
+{{- if $secretCA }}
+{{- $ca_key = index $secretCA.data "tls.key" }}
+{{- $ca_crt = index $secretCA.data "tls.crt" }}
+{{- end -}}
+{{- if and $secretCA $secretAuthzCert }}
+{{- $authz_key = index $secretAuthzCert.data "tls.key" }}
+{{- $authz_crt = index $secretAuthzCert.data "tls.crt" }}
+{{- end -}}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretCAName }}
+  labels:
+    app: {{ template "renku.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    "helm.sh/hook": "pre-install,pre-upgrade,pre-rollback"
+type: kubernetes.io/tls
+data:
+  tls.key: {{ $ca_key }}
+  tls.crt: {{ $ca_crt }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretAuthzCertName }}
+  labels:
+    app: {{ template "renku.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    "helm.sh/hook": "pre-install,pre-upgrade,pre-rollback"
+type: kubernetes.io/tls
+data:
+  tls.key: {{ $authz_key }}
+  tls.crt: {{ $authz_crt }}
diff --git a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml
@@ -143,7 +143,7 @@ spec:
             - name: DATASERVICE_KC_CLIENT_OAUTH_FLOW
               value: "client_credentials"
             - name: "DATASERVICE_KC_CLIENT_SERVICE_ACCOUNT_ROLES"
-              value: '["view-users", "query-users", "view-events"]'
+              value: '["view-users", "query-users", "view-events", "view-realm"]'
             - name: PYTHONUNBUFFERED
               value: "0"
             {{- include "certificates.env.python" . | nindent 12 }}

diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml
@@ -1597,12 +1597,12 @@ platformInit:
 dataService:
   image:
     repository: renku/renku-data-service
-    tag: "0.8.3"
+    tag: "0.10.0"
     pullPolicy: IfNotPresent
   keycloakSync:
     image:
       repository: renku/keycloak-sync
-      tag: "0.8.3"
+      tag: "0.10.0"
       pullPolicy: IfNotPresent
   service:
     type: ClusterIP
@@ -1645,7 +1645,7 @@ authz:
 secretsStorage:
   image:
     repository: renku/secrets-storage
-    tag: "v0.9.0"
+    tag: "v0.10.0"
     pullPolicy: IfNotPresent
   service:
     type: ClusterIP

diff --git a/scripts/init-realm/init-realm.py b/scripts/init-realm/init-realm.py
@@ -129,6 +129,7 @@ def _check_and_create_client(keycloak_admin, new_client: OIDCClient, force: bool
 
         keycloak_admin.delete_client(realm_client["id"])
         created_client_id = keycloak_admin.create_client(new_client.to_dict())
+        service_account_user = keycloak_admin.get_client_service_account_user(created_client_id)
 
         if isinstance(service_account_user, dict) and service_account_user.get("id"):
             logging.info(f"Reassigning service account roles {new_client.service_account_roles}")