diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3c37c9b94e..5de96c3708 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,47 @@ .. _changelog: +0.49.0 +------ + +The release contains bug fixes to renku core service related to project migration. + +This release also contains initial support for next generation 'Renku 2.0' functionality. However, +Renku 2.0 is still in early development and is not yet accessible to users. For more information, +see our [roadmap](https://github.com/SwissDataScienceCenter/renku-design-docs/blob/main/roadmap.md). + +**Note for administrators**: this release includes breaking changes due to upgrading PostgreSQL to 16.1.0. +This requires modifying the values file to work with the new PostgreSQL Helm chart. +Please check (`the helm chart values changelog `_) +for detailed instructions. + +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**🐞 Bug Fixes** + +- **Core Service**: Fix issue with having to run project migration twice to migrate the Dockerfile/project template. + +Internal Changes +~~~~~~~~~~~~~~~~ + +**New Features** + +- **Data services**: Initial support for Renku 2.0 projects (alpha release) + +**Improvements** + +- **csi-rclone**: added rclone logs to regular node-plugin logs. + (`#11 `_). + + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-python 2.9.2 `_ +- `renku-data-services 0.5.0 `_ +- `csi-rclone 0.1.7 `_ + + 0.48.1 ------ @@ -42,7 +84,8 @@ Internal Changes **Improvements** -- **UI**: Add initial alpha implementation of Renku 1.0 projects +- **Infrastructure**: Upgrade the version of PostgreSQL to 16.1.0. +- **UI**: Add initial alpha implementation of Renku 2.0 projects (`#2875 `_). Individual components @@ -51,7 +94,6 @@ Individual components - `renku-ui 3.20.1 `_ - `renku-ui 3.20.0 `_ - 0.47.1 ------ @@ -65,11 +107,11 @@ It doesn't bring any new features or bug fixes. This release expands Renku's cloud storage functionality in two key ways: First, mounted storages are now read **and write**, so you can use mounted storage as an active workspace for your data in a RenkuLab session. Second, we have expanded the cloud storage services you can integrate with RenkuLab. You can now -mount not only S3 buckets, but also WebDAV-based storages and Azure Blobs. - +mount not only S3 buckets, but also WebDAV-based storages and Azure Blobs. + If you use SSH sessions via the CLI, you can use cloud storage there too! Configure cloud storage for your project on RenkuLab.io, and those storages will be mounted in your remote session. Support for cloud -storage in local Renku sessions is still on our roadmap. +storage in local Renku sessions is still on our roadmap. This release also adds the ability to change which resource class your session uses when you unpause the session, in case the original resource class is now full. @@ -200,15 +242,15 @@ Internal Changes Individual components ~~~~~~~~~~~~~~~~~~~~~ -- `renku-python 2.8.2 `_ +- `renku-python 2.8.2 `_ - `renku-ui 3.17.2 `_ -- `renku-graph 2.48.2 `_ +- `renku-graph 2.48.2 `_ 0.45.1 ------ -This is a bugfix release that updates the helm chart to work with new -prometheus metrics in the renku core service, which was preventing it from +This is a bugfix release that updates the helm chart to work with new +prometheus metrics in the renku core service, which was preventing it from starting properly if metrics were enabled. In addition this release also addresses problems with expiring Gitlab access tokens when sessions are paused and resumed which caused resumed session to not be able to push to Gitlab @@ -219,8 +261,8 @@ Internal Changes **🐞 Bug Fixes** -- **Helm Chart**: update core-service deployment to allow service and rq - metrics to run side-by-side (`#3303 +- **Helm Chart**: update core-service deployment to allow service and rq + metrics to run side-by-side (`#3303 `_). - **Notebooks**: use a larger /dev/shm folder in sessions (`#1723 `_) diff --git a/RELEASE.md b/RELEASE.md index 08b6caeee1..f0aa7b3d49 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,7 +14,6 @@ This procedure should be followed for *any* release: * Create a `CHANGELOG` entry for the release and open a PR; create a deployment, this is the reference for the release. * Note that any PR that should go into the release needs to target the release branch _not_ `master`. * All release branches should be protected. -* Use the "Rebase and Merge" button to merge release branches into `master`; do not squash commits. Acceptance tests have to pass on all release branches before merging. diff --git a/acceptance-tests/.scalafmt.conf b/acceptance-tests/.scalafmt.conf index d37a95795b..0e06410bac 100644 --- a/acceptance-tests/.scalafmt.conf +++ b/acceptance-tests/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.7.17" +version = "3.8.0" runner.dialect = "scala213" diff --git a/acceptance-tests/README.md b/acceptance-tests/README.md index daa66cce9b..e2f231b8c7 100644 --- a/acceptance-tests/README.md +++ b/acceptance-tests/README.md @@ -204,7 +204,6 @@ The test are built using the Page Object Pattern (e.g. https://www.pluralsight.com/guides/getting-started-with-page-object-pattern-for-your-selenium-tests) which in short is about wrapping an UI page into a class/object and using it in the test script. - As mentioned above there's a `target/tests-execution.log` file where tests debug statements from tests execution are written. ## Project organization diff --git a/acceptance-tests/build.sbt b/acceptance-tests/build.sbt index a3d00290a8..19e984a6d7 100644 --- a/acceptance-tests/build.sbt +++ b/acceptance-tests/build.sbt @@ -19,7 +19,7 @@ organization := "ch.renku" name := "renku-acceptance-tests" version := "0.1.0" -scalaVersion := "2.13.12" +scalaVersion := "2.13.13" Test / parallelExecution := false @@ -30,8 +30,8 @@ publishTo := Some(Resolver.file("Unused transient repository", file("target/unus val circeVersion = "0.14.6" -libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.14" -libraryDependencies += "com.github.pureconfig" %% "pureconfig" % "0.17.5" % Test +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.5.3" +libraryDependencies += "com.github.pureconfig" %% "pureconfig" % "0.17.6" % Test libraryDependencies += "eu.timepit" %% "refined" % "0.11.1" % Test libraryDependencies += "io.circe" %% "circe-core" % circeVersion % Test libraryDependencies += "io.circe" %% "circe-literal" % circeVersion % Test @@ -43,7 +43,7 @@ libraryDependencies += "org.scalacheck" %% "scalacheck" libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.18" % Test libraryDependencies += "org.scalatestplus" %% "selenium-4-1" % "3.2.12.1" % Test libraryDependencies += "org.seleniumhq.selenium" % "selenium-http-jdk-client" % "4.13.0" % Test -libraryDependencies += "org.seleniumhq.selenium" % "selenium-java" % "4.17.0" % Test +libraryDependencies += "org.seleniumhq.selenium" % "selenium-java" % "4.18.1" % Test libraryDependencies += "org.slf4j" % "slf4j-log4j12" % "2.0.11" % Test libraryDependencies += "org.typelevel" %% "cats-effect" % "3.5.3" % Test libraryDependencies += "org.typelevel" %% "cats-effect-testing-scalatest" % "1.5.0" % Test diff --git a/acceptance-tests/project/build.properties b/acceptance-tests/project/build.properties index abbbce5da4..04267b14af 100644 --- a/acceptance-tests/project/build.properties +++ b/acceptance-tests/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.8 +sbt.version=1.9.9 diff --git a/chartpress.yaml b/chartpress.yaml index cc91c46588..1b55b12f21 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -19,6 +19,7 @@ charts: - helm-chart - acceptance-tests - scripts/init-realm + - scripts/init-db images: tests: buildArgs: diff --git a/docs/renku-python b/docs/renku-python index af89ce6ac1..74ccfb7149 160000 --- a/docs/renku-python +++ b/docs/renku-python @@ -1 +1 @@ -Subproject commit af89ce6ac12e00544158bacba87df82fdb03585f +Subproject commit 74ccfb714908c75747355d3ed27051b409c202b3 diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 6b13e25c57..b9b52d57cc 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -1,35 +1,34 @@ dependencies: -- name: gitlab - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: 0.8.0 - condition: gitlab.enabled -- name: postgresql - version: 9.1.1 - repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami" - condition: postgresql.enabled -- name: keycloakx - version: 2.1.0 - repository: "https://codecentric.github.io/helm-charts" - condition: keycloakx.enabled -- name: redis - # bitnami claims that this will always contain a full set of charts - let us pray... - # this index was 19MB as of the date of this commit and contained redis 17.4.2 - repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" - version: 17.4.2 - condition: redis.install -- name: renku-jena - version: "0.0.23" - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - alias: jena -- name: amalthea - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.11.0" -- name: dlf-chart - repository: "https://swissdatasciencecenter.github.io/datashim/" - version: "0.3.9-renku-2" - condition: notebooks.cloudstorage.s3.installDatashim -- name: csi-rclone - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.1.6" - condition: global.csi-rclone.install - + - name: gitlab + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: 0.8.0 + condition: gitlab.enabled + - name: postgresql + version: "14.2.4" + repository: "oci://registry-1.docker.io/bitnamicharts" + condition: postgresql.enabled + - name: keycloakx + version: 2.1.0 + repository: "https://codecentric.github.io/helm-charts" + condition: keycloakx.enabled + - name: redis + # bitnami claims that this will always contain a full set of charts - let us pray... + # this index was 19MB as of the date of this commit and contained redis 17.4.2 + repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" + version: 17.4.2 + condition: redis.install + - name: renku-jena + version: "0.0.23" + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + alias: jena + - name: amalthea + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.11.0" + - name: dlf-chart + repository: "https://swissdatasciencecenter.github.io/datashim/" + version: "0.3.9-renku-2" + condition: notebooks.cloudstorage.s3.installDatashim + - name: csi-rclone + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.1.7" + condition: global.csi-rclone.install diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl index 7930555279..b6c0a05df9 100644 --- a/helm-chart/renku/templates/_helpers.tpl +++ b/helm-chart/renku/templates/_helpers.tpl @@ -164,3 +164,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- define "renku.keycloak.realm" -}} {{ .Values.global.keycloak.realm | default "Renku" }} {{- end -}} + +{{- define "renku.dataService.keycloak.clientId" -}} +data-service +{{- end -}} diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index bed53c5f51..9c018bf973 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -58,6 +58,13 @@ spec: value: {{ (printf "%s://%s/auth/" (include "renku.http" .) .Values.global.renku.domain) | quote }} - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS value: "RS256" + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret - name: SERVER_DEFAULTS value: /etc/renku-data-service/server_options/server_defaults.json - name: SERVER_OPTIONS diff --git a/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml new file mode 100644 index 0000000000..c109f4875a --- /dev/null +++ b/helm-chart/renku/templates/keycloak-users-sync-cronjob.yaml @@ -0,0 +1,113 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "renku.fullname" . }}-keycloak-sync-events + labels: + app: keycloak-sync + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "renku.chart" . }} +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + app: keycloak-sync + spec: + initContainers: + {{- include "certificates.initContainer" . | nindent 12 }} + containers: + - name: keycloak-sync + image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}" + imagePullPolicy: IfNotPresent + env: + - name: DB_HOST + value: {{ template "postgresql.fullname" . }} + - name: DB_USER + value: {{ .Values.global.db.common.username }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.common.passwordSecretName }} + key: password + - name: KEYCLOAK_URL + value: {{ include "renku.keycloakUrl" . | quote}} + - name: KEYCLOAK_REALM + value: {{ include "renku.keycloak.realm" . | quote}} + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: TOTAL_USER_SYNC + value: "false" + {{- include "certificates.env.python" . | nindent 16 }} + volumeMounts: + {{- include "certificates.volumeMounts.system" . | nindent 16 }} + restartPolicy: Never + volumes: + {{- include "certificates.volumes" . | nindent 12 }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "renku.fullname" . }}-keycloak-sync-total + labels: + app: keycloak-sync + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "renku.chart" . }} +spec: + schedule: "0 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + app: keycloak-sync + spec: + initContainers: + {{- include "certificates.initContainer" . | nindent 12 }} + containers: + - name: keycloak-sync + image: "{{ .Values.dataService.keycloakSync.image.repository }}:{{ .Values.dataService.keycloakSync.image.tag }}" + imagePullPolicy: IfNotPresent + env: + - name: DB_HOST + value: {{ template "postgresql.fullname" . }} + - name: DB_USER + value: {{ .Values.global.db.common.username }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.common.passwordSecretName }} + key: password + - name: KEYCLOAK_URL + value: {{ include "renku.keycloakUrl" . | quote}} + - name: KEYCLOAK_REALM + value: {{ include "renku.keycloak.realm" . | quote}} + - name: KEYCLOAK_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: TOTAL_USER_SYNC + value: "true" + {{- include "certificates.env.python" . | nindent 16 }} + volumeMounts: + {{- include "certificates.volumeMounts.system" . | nindent 16 }} + restartPolicy: Never + volumes: + {{- include "certificates.volumes" . | nindent 12 }} diff --git a/helm-chart/renku/templates/network-policies.yaml b/helm-chart/renku/templates/network-policies.yaml index b2c77647ef..431f5fbd6d 100644 --- a/helm-chart/renku/templates/network-policies.yaml +++ b/helm-chart/renku/templates/network-policies.yaml @@ -69,6 +69,12 @@ spec: namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} + - podSelector: + matchLabels: + app: keycloak-sync + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} ports: - protocol: TCP port: 5432 diff --git a/helm-chart/renku/templates/secrets.yaml b/helm-chart/renku/templates/secrets.yaml index 0b39dfac76..e8bd0640bb 100644 --- a/helm-chart/renku/templates/secrets.yaml +++ b/helm-chart/renku/templates/secrets.yaml @@ -1,4 +1,14 @@ --- +{{- $data_service_kc_client_secret := (randAlphaNum 64) | b64enc | quote }} + +{{- $renku_secret := lookup "v1" "Secret" .Release.Namespace (include "renku.fullname" .) }} +{{- if and $renku_secret $renku_secret.data }} +{{- $data_service_kc_client_secret_test := index $renku_secret.data "dataServiceKeycloakClientSecret" }} +{{- if $data_service_kc_client_secret_test }} +{{- $data_service_kc_client_secret = $data_service_kc_client_secret_test }} +{{- end -}} +{{- end -}} + apiVersion: v1 kind: Secret metadata: @@ -8,11 +18,16 @@ metadata: chart: {{ template "renku.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + annotations: + # If "keep" resource policy is removed the secret is deleted post upgrade see https://github.com/helm/helm/issues/8420 + "helm.sh/resource-policy": keep + "helm.sh/hook": "pre-install,pre-upgrade,pre-rollback" type: Opaque data: {{- if .Values.tests.users_json }} users.json: {{ .Values.tests.users_json | toJson | b64enc | quote }} {{- end }} + dataServiceKeycloakClientSecret: {{ $data_service_kc_client_secret }} {{- if and (eq .Values.redis.install true) (eq .Values.redis.createSecret true) }} diff --git a/helm-chart/renku/templates/setup-job-gitlab.yaml b/helm-chart/renku/templates/setup-job-gitlab.yaml index f01a4747d5..b6fc4cc58c 100644 --- a/helm-chart/renku/templates/setup-job-gitlab.yaml +++ b/helm-chart/renku/templates/setup-job-gitlab.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: GITLAB_ENABLED value: {{ .Values.gitlab.enabled | quote }} diff --git a/helm-chart/renku/templates/setup-job-keycloak-db.yaml b/helm-chart/renku/templates/setup-job-keycloak-db.yaml index 4d3229b2f0..68736d5441 100644 --- a/helm-chart/renku/templates/setup-job-keycloak-db.yaml +++ b/helm-chart/renku/templates/setup-job-keycloak-db.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: KEYCLOAK_DB_USERNAME value: {{ .Values.global.keycloak.postgresUser | quote }} diff --git a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml index 0efc5d88fc..84735d221c 100644 --- a/helm-chart/renku/templates/setup-job-keycloak-realms.yaml +++ b/helm-chart/renku/templates/setup-job-keycloak-realms.yaml @@ -86,6 +86,8 @@ spec: key: oidcClientSecret - name: RENKU_KC_CLIENT_PUBLIC value: "false" + - name: RENKU_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: CLI_KC_CLIENT_ID value: renku-cli - name: CLI_KC_CLIENT_SECRET @@ -97,6 +99,8 @@ spec: value: "true" - name: CLI_KC_CLIENT_ATTRIBUTES value: '{"access.token.lifespan": "86400", "oauth2.device.authorization.grant.enabled": true, "oauth2.device.polling.interval": "5"}' + - name: CLI_KC_CLIENT_OAUTH_FLOW + value: "device" - name: UI_KC_CLIENT_ID value: "renku-ui" - name: UI_KC_CLIENT_SECRET @@ -106,6 +110,8 @@ spec: key: uiserverClientSecret - name: UI_KC_CLIENT_PUBLIC value: "false" + - name: UI_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: NOTEBOOKS_KC_CLIENT_ID value: {{ .Values.notebooks.oidc.clientId | default "renku-jupyterserver" | quote }} - name: NOTEBOOKS_KC_CLIENT_SECRET @@ -115,12 +121,29 @@ spec: key: notebooksClientSecret - name: NOTEBOOKS_KC_CLIENT_PUBLIC value: "false" + - name: NOTEBOOKS_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: SWAGGER_KC_CLIENT_ID value: swagger - name: SWAGGER_KC_CLIENT_PUBLIC value: "true" + - name: SWAGGER_KC_CLIENT_OAUTH_FLOW + value: "authorization_code" - name: SWAGGER_KC_CLIENT_ATTRIBUTES value: '{"pkce.code.challenge.method": "S256"}' + - name: DATASERVICE_KC_CLIENT_ID + value: {{ include "renku.dataService.keycloak.clientId" . | quote }} + - name: DATASERVICE_KC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "renku.fullname" . }} + key: dataServiceKeycloakClientSecret + - name: DATASERVICE_KC_CLIENT_PUBLIC + value: "false" + - name: DATASERVICE_KC_CLIENT_OAUTH_FLOW + value: "client_credentials" + - name: "DATASERVICE_KC_CLIENT_SERVICE_ACCOUNT_ROLES" + value: '["view-users", "query-users", "view-events"]' - name: PYTHONUNBUFFERED value: "0" {{- include "certificates.env.python" . | nindent 12 }} diff --git a/helm-chart/renku/templates/setup-job-renku-dbs.yaml b/helm-chart/renku/templates/setup-job-renku-dbs.yaml index 50a49b7a7a..45dda94363 100644 --- a/helm-chart/renku/templates/setup-job-renku-dbs.yaml +++ b/helm-chart/renku/templates/setup-job-renku-dbs.yaml @@ -34,14 +34,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -50,7 +50,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: EVENTLOG_DB_USERNAME value: {{ .Values.global.graph.dbEventLog.postgresUser | quote }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index b0ae4d8269..3fa5dec831 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -62,7 +62,7 @@ global: fullnameOverride: "" image: repository: renku/renku-core - tag: "v2.9.1" + tag: "v2.9.2" pullPolicy: IfNotPresent uiserver: ## The client secret for the renku-ui client application registered in keycloak. @@ -334,36 +334,43 @@ keycloakx: # For production deployments check out # https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values-production.yaml postgresql: - # If an external Postgres database is defined in global.externalServices.postgresql, - # postgresql.enabled should be false, and global.externalServices.postgresql.enabled should be true. - # By default, Renku-bundled Postgres is enabled. - enabled: true - ## We use the defaults here. - postgresqlDatabase: postgres - postgresqlUsername: postgres - ## The admin password should be set explicitly, otherwise a random string will be created. - ## Alternatively an existing secret can be provided. Note that postgres - ## DOES NOT tolerate a change of the admin password when upgrading. - # postgresqlPassword: + ## We use the defaults here. Note that these basic configs could also be set as + ## global values such that sub-charts can access them too. + + auth: + username: postgres + database: postgres + + ## The admin password should be set explicitly, otherwise a random string will be + ## created. Alternatively an existing secret can be provided. Note that postgres + ## DOES NOT tolerate a change of the admin password when upgrading. + # postgresqlPassword: + + ## Use an existing secret instead of creating a new one. It must have a + ## postgresql-password key containing the password for the postgres user. + # existingSecret: + + # Consider replication. These are the defaults for the basic settings. + # replicationUsername: repl_user + # replicationPassword: repl_password # generate a random password `openssl rand -hex 32` + + # image: + # repository: bitnami/postgresql + # tag: + + primary: + persistence: + ## We use the defaults here, but they will probably be modified for most deployments. + enabled: true + size: 8Gi + ## Provide an existing PersistentVolumeClaim to be reused. + # existingClaim: + + # Consider replication. These are the defaults for the basic settings. + readReplicas: + enabled: false + replicaCount: 1 - ## Use an existing secret instead of creating a new one. It must have a - ## postgresql-password key containing the password for the posgres user. - # existingSecret: - image: - repository: bitnami/postgresql - tag: 12.8.0 - persistence: - ## We use the defaults here, but they will probably be modified for most deployments. - enabled: true - size: 8Gi - ## Provide an existing PersistentVolumeClaim to be reused. - # existingClaim: - # Consider replication. These are the defaults for the basic settings. - replication: - enabled: false - user: repl_user - password: repl_password # generate a random password `openssl rand -hex 32` - slaveReplicas: 1 redis: # If set to true, a HA redis will be included in the Renku release. install: true @@ -854,7 +861,8 @@ dlf-chart: enabled: false dataset-operator-chart: enabled: true -csi-rclone: {} +csi-rclone: + {} # This section is only relevant if you are installing csi-rclone as part of Renku ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster. # storageClassName: csi-rclone @@ -1480,8 +1488,13 @@ initDb: dataService: image: repository: renku/renku-data-service - tag: "0.4.0" + tag: "0.5.0" pullPolicy: IfNotPresent + keycloakSync: + image: + repository: renku/keycloak-sync + tag: "0.5.0" + pullPolicy: IfNotPresent service: type: ClusterIP port: 80 @@ -1512,4 +1525,4 @@ affinity: {} versions: latest: image: - tag: v2.9.1 + tag: v2.9.2 diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml index 393c415139..c630909dbf 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml @@ -27,7 +27,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password ports: - containerPort: 5432 volumeMounts: @@ -81,7 +81,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml index d4395019f2..1ad02e0d18 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml @@ -28,7 +28,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /bitnami/postgresql name: pg-vol-new @@ -80,7 +80,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index 41fdf5ee1d..d5f7ea6afd 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,51 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.49.0 + +The PostgreSQL chart dependency has been upgraded, which requires modification of the postgres data volume of existing deployments. See [these instructions](https://github.com/SwissDataScienceCenter/renku/tree/master/helm-chart/utils/postgres_migrations/version_upgrades/README.md) for more details. + +* NEW/EDIT - *postgresql.persistence.existingClaim*: Renku `0.xx.0` upgrades the postgres chart dependency, which requires modification of the postgres data volume of existing deployments. See [these instructions](https://github.com/SwissDataScienceCenter/renku/tree/master/helm-chart/utils/postgres_migrations/version_upgrades/README.md) + +* EDIT - *postgresql*: The upgrade of the postgres chart dependency requires some restructuring of the postgres subchart values to match those of bitnami/postgresql chart version 14.0.1, namely: + +Old + ``` + postgresql: + postgresqlDatabase: + postgresqlUsername: + postgresqlPassword: + existingSecret: + persistence: + enabled: + size: + existingClaim: + replication: + enabled: + user: + password: + slaveReplicas: + ``` +New + ``` + postgresql: + auth: + username: + database: + postgresqlPassword: + existingSecret: + replicationUsername: + replicationPassword: + primary: + persistence: + enabled: + size: + existingClaim: + readReplicas: + enabled: + replicaCount: + ``` + ## Upgrading to Renku 0.48.1 The handling of privacy policy and terms of service content has been fine tuned. @@ -27,7 +72,6 @@ it more flexible. * NEW `ui.client.privacy.page.configMapPolicyKey` the key in the ConfigMap where the content for the privacy policy is located. * NEW `ui.client.privacy.page.configMapTermsKey` the key in the ConfigMap where the content for the terms of use is located. - ## Upgrading to Renku 0.47.0 We completely overhauled how mounting cloud storage in sessions works, relying on a new CSI driver based on RClone diff --git a/scripts/init-realm/Dockerfile b/scripts/init-realm/Dockerfile index 2fa09059cf..8ed86a2417 100644 --- a/scripts/init-realm/Dockerfile +++ b/scripts/init-realm/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine COPY requirements.txt init-realm.py utils.py /app/ WORKDIR /app diff --git a/scripts/init-realm/init-realm.py b/scripts/init-realm/init-realm.py index ea784e6a9c..ec3bac706c 100644 --- a/scripts/init-realm/init-realm.py +++ b/scripts/init-realm/init-realm.py @@ -21,6 +21,7 @@ import json import time import logging +import os from typing import Dict, List from keycloak import KeycloakAdmin @@ -30,7 +31,7 @@ KeycloakPostError, ) -from utils import DemoUserConfig, OIDCClientsConfig, OIDCGitlabClient +from utils import DemoUserConfig, OIDCClientsConfig, OIDCGitlabClient, OIDCClient, OAuthFlow logging.basicConfig(level=logging.INFO) @@ -70,18 +71,19 @@ def _fix_json_values(data: Dict) -> Dict: return json.loads(json.dumps(data).replace('"true"', "true").replace('"false"', "false")) -def _check_and_create_client(keycloak_admin, new_client, force: bool): +def _check_and_create_client(keycloak_admin, new_client: OIDCClient, force: bool): """ Check if a client exists. Create it if not. Alert if it exists but with different details than what is provided. """ - logging.info("Checking if {} client exists...".format(new_client["clientId"])) + logging.info("Checking if {} client exists...".format(new_client.id)) realm_clients = keycloak_admin.get_clients() client_ids = [c["clientId"] for c in realm_clients] - if new_client["clientId"] in client_ids: + realm_management_client_id = keycloak_admin.get_client_id("realm-management") + if new_client.id in client_ids: logging.info("found") - realm_client = realm_clients[client_ids.index(new_client["clientId"])] + realm_client = realm_clients[client_ids.index(new_client.id)] # We have to separately query the secret as it is not part of # the original response @@ -99,22 +101,57 @@ def _check_and_create_client(keycloak_admin, new_client, force: bool): if "attributes" in realm_client: realm_client["attributes"] = _fix_json_values(realm_client["attributes"]) - changed = _check_existing(realm_client, new_client, "client", "clientId") - - if not force or not changed: + roles_changed = False + service_account_user = None + existing_roles = [] + if new_client.oauth_flow == OAuthFlow.client_credentials: + try: + service_account_user = keycloak_admin.get_client_service_account_user(realm_client["id"]) + except KeycloakGetError as err: + if err.response_code != 404: + raise + if isinstance(service_account_user, dict): + try: + existing_roles = keycloak_admin.get_client_roles_of_user(service_account_user["id"], realm_management_client_id) + except KeycloakGetError as err: + if err.response_code != 404: + raise + existing_roles_names = [role["name"] for role in existing_roles] + if set(existing_roles_names) != set(new_client.service_account_roles): + logging.warning(f"Roles changed existing roles {set(existing_roles_names)} != new roles {set(new_client.service_account_roles)}") + roles_changed = True + changed = _check_existing(realm_client, new_client.to_dict(), "client", "clientId") + + if not force or (not changed and not roles_changed): return logging.info(f"Recreating modified client '{realm_client['clientId']}'...") keycloak_admin.delete_client(realm_client["id"]) - keycloak_admin.create_client(new_client) + created_client_id = keycloak_admin.create_client(new_client.to_dict()) + + if isinstance(service_account_user, dict) and service_account_user.get("id"): + logging.info(f"Reassigning service account roles {new_client.service_account_roles}") + realm_management_roles = keycloak_admin.get_client_roles(realm_management_client_id) + matching_roles = [{"name": role["name"], "id": role["id"]} for role in realm_management_roles if role["name"] in new_client.service_account_roles ] + logging.info(f"Found and assigning matching roles: {matching_roles}") + keycloak_admin.assign_client_role(service_account_user["id"], realm_management_client_id, matching_roles) logging.info("done") else: logging.info("not found") - logging.info("Creating {} client...".format(new_client["clientId"])) - keycloak_admin.create_client(new_client) + logging.info("Creating {} client...".format(new_client.id)) + created_client_id = keycloak_admin.create_client(new_client.to_dict()) + if new_client.oauth_flow == OAuthFlow.client_credentials and new_client.service_account_roles: + service_account_user = keycloak_admin.get_client_service_account_user(created_client_id) + logging.info(f"Assigning service account roles {new_client.service_account_roles}") + realm_management_client_id = keycloak_admin.get_client_id("realm-management") + realm_management_roles = keycloak_admin.get_client_roles(realm_management_client_id) + matching_roles = [{"name": role["name"], "id": role["id"]} for role in realm_management_roles if role["name"] in new_client.service_account_roles ] + logging.info(f"Found and assigning matching roles: {matching_roles}") + keycloak_admin.assign_client_role(service_account_user["id"], realm_management_client_id, matching_roles) + logging.info("done") @@ -234,15 +271,35 @@ def _check_and_create_user(keycloak_admin, new_user): ) logging.info("done") +realm = keycloak_admin.get_realm(args.realm) +event_retention_seconds = 86400 +if not realm.get("eventsEnabled"): + logging.info( + f"Enabling user events tracking for realm with retention {event_retention_seconds}" + ) + keycloak_admin.update_realm(args.realm, {"eventsEnabled": True, "eventsExpiration": event_retention_seconds}) +if not realm.get("adminEventsEnabled"): + logging.info( + f"Enabling admin events tracking for realm with retention {event_retention_seconds}" + ) + keycloak_admin.update_realm( + args.realm, + { + "adminEventsEnabled": True, + "adminEventsDetailsEnabled": True, + "attributes": {"adminEventsExpiration": event_retention_seconds}, + }, + ) + # Switching to the newly created realm keycloak_admin.connection.realm_name = args.realm -for new_client in OIDCClientsConfig.from_env().to_list(): - _check_and_create_client(keycloak_admin, new_client, args.force) +for client in OIDCClientsConfig.from_env().to_list(): + _check_and_create_client(keycloak_admin, client, args.force) -gitlab_oidc_client = OIDCGitlabClient.from_env().to_dict() -if gitlab_oidc_client is not None: +if os.environ.get("INTERNAL_GITLAB_ENABLED", "false").lower() == "true": + gitlab_oidc_client = OIDCGitlabClient.from_env() _check_and_create_client(keycloak_admin, gitlab_oidc_client, args.force) # Create renku-admin realm role diff --git a/scripts/init-realm/requirements.txt b/scripts/init-realm/requirements.txt index 1a933ca698..7893b521ac 100644 --- a/scripts/init-realm/requirements.txt +++ b/scripts/init-realm/requirements.txt @@ -1 +1 @@ -python-keycloak==3.7.0 +python-keycloak==3.9.1 diff --git a/scripts/init-realm/utils.py b/scripts/init-realm/utils.py index ff009e5930..63501d58bf 100644 --- a/scripts/init-realm/utils.py +++ b/scripts/init-realm/utils.py @@ -1,6 +1,8 @@ import json import os +from copy import deepcopy from dataclasses import dataclass, field +from enum import Enum from typing import Any, Dict, List, Optional @@ -36,6 +38,39 @@ def to_dict(self) -> Optional[Dict[str, Any]]: } +class OAuthFlow(Enum): + device: str = "device" + authorization_code: str = "authorization_code" + client_credentials: str = "client_credentials" + + def get_keycloak_payload( + self, + existing_payload: Dict[str, Any] | None = None, + disable_other_flows: bool = True + ) -> Dict[str, Any]: + output = deepcopy(existing_payload) if existing_payload else {} + if disable_other_flows: + output.update( + serviceAccountsEnabled=False, + standardFlowEnabled=False, + ) + match self: + case OAuthFlow.authorization_code: + output["standardFlowEnabled"] = True + case OAuthFlow.device: + if isinstance(output.get("attributes"), dict): + output["attributes"]["oauth2.device.authorization.grant.enabled"] = True + else: + output["attributes"] = {"oauth2.device.authorization.grant.enabled": True} + case OAuthFlow.client_credentials: + output["serviceAccountsEnabled"] = True + return output + + @classmethod + def from_env(cls, prefix: str = ""): + return cls(os.environ.get(f"{prefix}OAUTH_FLOW")) + + @dataclass class OIDCClient: """Stores the configuration needed to create an OIDC client application in Keycloak. These @@ -44,8 +79,11 @@ class OIDCClient: id: str base_url: str + oauth_flow: OAuthFlow + disable_other_oauth_flows: bool = True secret: Optional[str] = field(default=None, repr=False) attributes: Dict[str, Any] = field(default_factory=lambda: {}) + service_account_roles: List[str] = field(default_factory=list) public_client: bool = False def __post_init__(self): @@ -55,8 +93,58 @@ def __post_init__(self): f"The OIDC client configuration for client {self.id} is not valid, " "the client is marked as not public but a secret is not provided." ) + if self.oauth_flow != OAuthFlow.client_credentials and len(self.service_account_roles) > 0: + raise ValueError( + f"Service account roles can only be specified for the {OAuthFlow.client_credentials.value} flow" + ) def to_dict(self) -> Dict[str, Any]: + default_protocol_mappers = [] + if self.oauth_flow == OAuthFlow.client_credentials: + default_protocol_mappers.extend([ + { + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientId", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientHost", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientHost", + "jsonType.label": "String" + } + }, + { + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": False, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": True, + "access.token.claim": True, + "userinfo.token.claim": True, + "claim.name": "clientAddress", + "jsonType.label": "String" + } + } + ]) output = { "clientId": self.id, "baseUrl": self.base_url, @@ -64,7 +152,7 @@ def to_dict(self) -> Dict[str, Any]: "attributes": self.attributes, "redirectUris": [self.base_url + "/*"], "webOrigins": [self.base_url + "/*"], - "protocolMappers": [ + "protocolMappers": default_protocol_mappers + [ { "name": "renku audience for renku cli", "protocol": "openid-connect", @@ -81,6 +169,7 @@ def to_dict(self) -> Dict[str, Any]: } if self.secret is not None: output["secret"] = self.secret + output = self.oauth_flow.get_keycloak_payload(output, self.disable_other_oauth_flows) return output @classmethod @@ -91,43 +180,34 @@ def from_env(cls, prefix: str = "RENKU_KC_CLIENT_") -> "OIDCClient": base_url=os.environ.get(f"{prefix}BASE_URL", os.environ["RENKU_BASE_URL"]), attributes=json.loads(os.environ.get(f"{prefix}ATTRIBUTES", "{}")), public_client=os.environ.get(f"{prefix}PUBLIC", "false").lower() == "true", + oauth_flow=OAuthFlow.from_env(prefix), + disable_other_oauth_flows=os.environ.get( + f"{prefix}DISABLE_OTHER_OAUTH_FLOWS", "true" + ).lower() == "true", + service_account_roles=json.loads(os.environ.get(f"{prefix}SERVICE_ACCOUNT_ROLES", "[]")), ) @dataclass -class OIDCGitlabClient: - """A Keycloak OIDC client used by the internal Renku Gitlab deployment (if this deployment is enabled).""" - - internal_gitlab_enabled: bool = False - oidc_client_secret: Optional[str] = field(default=None, repr=False) - oidc_client_id: str = "gitlab" - renku_base_url: Optional[str] = None - - def __post_init__(self): - if self.internal_gitlab_enabled and not (self.oidc_client_secret or self.renku_base_url): - raise ValueError( - "The internal Gitlab is enabled, but the Renku base URL and/or the Keycloak OIDC client secret are not defined." - ) - self.renku_base_url = self.renku_base_url.rstrip("/") +class OIDCGitlabClient(OIDCClient): + """A Keycloak OIDC client used by the internal Renku Gitlab deployment.""" @classmethod def from_env(cls, prefix: str = "INTERNAL_GITLAB_") -> "OIDCGitlabClient": return cls( - internal_gitlab_enabled=os.environ.get(f"{prefix}ENABLED", "false").lower() == "true", - oidc_client_secret=os.environ.get(f"{prefix}OIDC_CLIENT_SECRET"), - oidc_client_id=os.environ.get(f"{prefix}OIDC_CLIENT_ID", "gitlab"), - renku_base_url=os.environ.get(f"RENKU_BASE_URL"), + secret=os.environ.get(f"{prefix}OIDC_CLIENT_SECRET"), + id=os.environ.get(f"{prefix}OIDC_CLIENT_ID", "gitlab"), + base_url=os.environ.get("RENKU_BASE_URL"), + oauth_flow=OAuthFlow.authorization_code, ) def to_dict(self) -> Optional[Dict[str, Any]]: - if not self.internal_gitlab_enabled: - return None return { - "clientId": self.oidc_client_id, - "baseUrl": f"{self.renku_base_url}/gitlab", - "secret": self.oidc_client_secret, + "clientId": self.id, + "baseUrl": f"{self.base_url}", + "secret": self.secret, "redirectUris": [ - f"{self.renku_base_url}/gitlab/users/auth/oauth2_generic/callback", + f"{self.base_url}/users/auth/oauth2_generic/callback", ], "webOrigins": [], } @@ -140,6 +220,7 @@ class OIDCClientsConfig: ui: OIDCClient notebooks: OIDCClient swagger: OIDCClient + data_service: OIDCClient @classmethod def from_env(cls) -> "OIDCClientsConfig": @@ -149,13 +230,15 @@ def from_env(cls) -> "OIDCClientsConfig": ui=OIDCClient.from_env(prefix="UI_KC_CLIENT_"), notebooks=OIDCClient.from_env(prefix="NOTEBOOKS_KC_CLIENT_"), swagger=OIDCClient.from_env(prefix="SWAGGER_KC_CLIENT_"), + data_service=OIDCClient.from_env(prefix="DATASERVICE_KC_CLIENT_"), ) - def to_list(self) -> List[Dict[str, Any]]: + def to_list(self) -> List[OIDCClient]: return [ - self.renku.to_dict(), - self.cli.to_dict(), - self.ui.to_dict(), - self.notebooks.to_dict(), - self.swagger.to_dict(), + self.renku, + self.cli, + self.ui, + self.notebooks, + self.swagger, + self.data_service, ]