From 0f75705e4746eb6ab940978c6a1494383eeb1830 Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Fri, 21 Jun 2024 08:12:53 +0000 Subject: [PATCH 01/42] chore: create release 0.55.0 --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f7d693546f..948fc32ee1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,9 @@ .. _changelog: +0.55.0 +------ + + 0.54.0 ------ From a927eced80d144634c09b8fa962838b3ad4ca3eb Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 15 Jul 2024 13:20:02 +0200 Subject: [PATCH 02/42] feat: jupyter free sessions --- .../templates/data-service/deployment.yaml | 3 + .../renku/templates/notebooks/env-secret.yaml | 87 +++++++++++ .../templates/notebooks/statefulset.yaml | 136 +----------------- 3 files changed, 93 insertions(+), 133 deletions(-) create mode 100644 helm-chart/renku/templates/notebooks/env-secret.yaml diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index a0620c6f78..ed9abf3857 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -43,6 +43,9 @@ spec: - name: http containerPort: 8000 protocol: TCP + envFrom: + - secretRef: + name: {{ template "renku.notebooks.fullname" . }} env: - name: VERSION value: {{ .Values.dataService.image.tag | quote }} diff --git a/helm-chart/renku/templates/notebooks/env-secret.yaml b/helm-chart/renku/templates/notebooks/env-secret.yaml new file mode 100644 index 0000000000..03bf4e2851 --- /dev/null +++ b/helm-chart/renku/templates/notebooks/env-secret.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "renku.notebooks.fullname" . }} + labels: + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +stringData: + NB_SESSIONS__STORAGE__PVS_ENABLED: {{ .Values.notebooks.userSessionPersistentVolumes.enabled | quote }} + {{ if .Values.notebooks.userSessionPersistentVolumes.enabled }} + NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS: {{ .Values.notebooks.userSessionPersistentVolumes.storageClass | quote}} + {{ end }} + NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT: {{ .Values.notebooks.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }} + NB_SESSIONS__DEFAULT_IMAGE: "{{ .Values.notebooks.defaultSessionImage }}" + NB_SERVER_OPTIONS__DEFAULTS_PATH: /etc/renku-notebooks/server_options/server_defaults.json + NB_SERVER_OPTIONS__UI_CHOICES_PATH: /etc/renku-notebooks/server_options/server_options.json + NB_SESSIONS__OIDC__CLIENT_ID: {{ .Values.notebooks.oidc.clientId }} + NB_SESSIONS__OIDC__CLIENT_SECRET: {{ .Values.notebooks.oidc.clientSecret }} + NB_SESSIONS__OIDC__AUTH_URL: {{ .Values.notebooks.oidc.authUrl }} + NB_SESSIONS__OIDC__TOKEN_URL: {{ .Values.notebooks.oidc.tokenUrl }} + NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }} + NB_SESSIONS__INGRESS__HOST: {{ .Values.notebooks.sessionIngress.host }} + NB_SESSIONS__INGRESS__TLS_SECRET: {{ .Values.notebooks.sessionIngress.tlsSecret }} + NB_SESSIONS__INGRESS__ANNOTATIONS: | + {{- .Values.notebooks.sessionIngress.annotations | toYaml | nindent 4 }} + NB_GIT__URL: {{ .Values.global.gitlab.url | quote }} + NB_GIT__REGISTRY: {{ required "An image registry must be specified." .Values.global.gitlab.registry.host }} + NB_SESSIONS__GIT_RPC_SERVER__IMAGE: "{{ .Values.notebooks.gitRpcServer.image.name }}:{{ .Values.notebooks.gitRpcServer.image.tag }}" + NB_SESSIONS__GIT_PROXY__IMAGE: "{{ .Values.notebooks.gitHttpsProxy.image.name }}:{{ .Values.notebooks.gitHttpsProxy.image.tag }}" + NB_SESSIONS__GIT_CLONE__IMAGE: "{{ .Values.notebooks.gitClone.image.name }}:{{ .Values.notebooks.gitClone.image.tag }}" + NB_ANONYMOUS_SESSIONS_ENABLED: {{ .Values.global.anonymousSessions.enabled | quote }} + NB_SSH_ENABLED: {{ .Values.notebooks.ssh.enabled | quote }} + NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.anonymous | quote }} + NB_SESSIONS__CULLING__REGISTERED__HIBERNATED_SECONDS: {{ .Values.notebooks.culling.hibernatedSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.anonymous | quote }} + NB_AMALTHEA__GROUP: {{ .Values.amalthea.crdApiGroup }} + NB_AMALTHEA__VERSION: {{ .Values.amalthea.crdApiVersion }} + NB_AMALTHEA__PLURAL: {{ .Values.amalthea.crdNames.plural }} + NB_AMALTHEA__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher" + NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.gitClone.sampleRate | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.sidecar.enabled | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.sidecar.dsn | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.sidecar.environment | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.sidecar.sampleRate | quote }} + NB_SESSIONS__CA_CERTS__IMAGE: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}" + NB_SESSIONS__CA_CERTS__SECRETS: | + {{- .Values.global.certificates.customCAs | toYaml | nindent 4 }} + {{- with .Values.notebooks.sessionNodeSelector }} + NB_SESSIONS__NODE_SELECTOR: | + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.notebooks.sessionAffinity }} + NB_SESSIONS__AFFINITY: | + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.notebooks.sessionTolerations }} + NB_SESSIONS__TOLERATIONS: | + {{- toYaml . | nindent 4 }} + {{- end }} + NB_SESSIONS__ENFORCE_CPU_LIMITS: {{ .Values.notebooks.enforceCPULimits | quote }} + NB_CLOUD_STORAGE__ENABLED: {{ .Values.notebooks.cloudstorage.enabled | quote }} + NB_CLOUD_STORAGE__STORAGE_CLASS: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} + NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} + NB_VERSION: {{ .Values.notebooks.image.tag | quote }} + {{ if .Values.notebooks.sessionsNamespace }} + NB_K8S__SESSIONS_NAMESPACE: {{ .Values.notebooks.sessionsNamespace | quote }} + {{ end }} + NB_K8S__RENKU_NAMESPACE: {{ .Release.Namespace | quote }} + NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_ID: renku + NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_SECRET: {{ .Values.global.gateway.clientSecret | quote }} + NB_KEYCLOAK_REALM: {{ include "renku.keycloak.realm" . | quote }} + NB_SESSIONS__SSH__ENABLED: {{ .Values.notebooks.ssh.enabled | quote }} + {{- if not (kindIs "invalid" .Values.notebooks.ssh.hostKeySecret) }} + NB_SESSIONS__SSH__HOST_KEY_SECRET: {{ .Values.notebooks.ssh.hostKeySecret | quote }} + {{- end }} + NB_DATA_SERVICE_URL: {{ printf "http://%s-data-service/api/data" .Release.Name }} + NB_USER_SECRETS__SECRETS_STORAGE_SERVICE_URL: {{ printf "http://%s-secrets-storage" .Release.Name }} + NB_USER_SECRETS__IMAGE: "{{ .Values.notebooks.secretsMount.image.repository}}:{{.Values.notebooks.secretsMount.image.tag }}" +--- + diff --git a/helm-chart/renku/templates/notebooks/statefulset.yaml b/helm-chart/renku/templates/notebooks/statefulset.yaml index f800651ec3..04a887f900 100644 --- a/helm-chart/renku/templates/notebooks/statefulset.yaml +++ b/helm-chart/renku/templates/notebooks/statefulset.yaml @@ -32,74 +32,14 @@ spec: imagePullPolicy: {{ .Values.notebooks.image.pullPolicy }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + envFrom: + - secretRef: + name: {{ template "renku.notebooks.fullname" . }} env: - - name: NB_SESSIONS__STORAGE__PVS_ENABLED - value: {{ .Values.notebooks.userSessionPersistentVolumes.enabled | quote }} - {{ if .Values.notebooks.userSessionPersistentVolumes.enabled }} - - name: NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS - value: {{ .Values.notebooks.userSessionPersistentVolumes.storageClass | quote}} - {{ end }} - - name: NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT - value: {{ .Values.notebooks.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }} - - name: NB_SESSIONS__DEFAULT_IMAGE - value: "{{ .Values.notebooks.defaultSessionImage }}" - - name: NB_SERVER_OPTIONS__DEFAULTS_PATH - value: /etc/renku-notebooks/server_options/server_defaults.json - - name: NB_SERVER_OPTIONS__UI_CHOICES_PATH - value: /etc/renku-notebooks/server_options/server_options.json {{ if eq .Values.global.debug true }} - name: FLASK_DEBUG value: "1" {{ end }} - - name: NB_SESSIONS__OIDC__CLIENT_ID - value: {{ .Values.notebooks.oidc.clientId }} - - name: NB_SESSIONS__OIDC__CLIENT_SECRET - value: {{ .Values.notebooks.oidc.clientSecret }} - - name: NB_SESSIONS__OIDC__AUTH_URL - value: {{ .Values.notebooks.oidc.authUrl }} - - name: NB_SESSIONS__OIDC__TOKEN_URL - value: {{ .Values.notebooks.oidc.tokenUrl }} - - name: NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL - value: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }} - - name: NB_SESSIONS__INGRESS__HOST - value: {{ .Values.notebooks.sessionIngress.host }} - - name: NB_SESSIONS__INGRESS__TLS_SECRET - value: {{ .Values.notebooks.sessionIngress.tlsSecret }} - - name: NB_SESSIONS__INGRESS__ANNOTATIONS - value: | - {{- .Values.notebooks.sessionIngress.annotations | toYaml | nindent 16 }} - - name: NB_GIT__URL - value: {{ .Values.global.gitlab.url | quote }} - - name: NB_GIT__REGISTRY - value: {{ required "An image registry must be specified." .Values.global.gitlab.registry.host }} - - name: NB_SESSIONS__GIT_RPC_SERVER__IMAGE - value: "{{ .Values.notebooks.gitRpcServer.image.name }}:{{ .Values.notebooks.gitRpcServer.image.tag }}" - - name: NB_SESSIONS__GIT_PROXY__IMAGE - value: "{{ .Values.notebooks.gitHttpsProxy.image.name }}:{{ .Values.notebooks.gitHttpsProxy.image.tag }}" - - name: NB_SESSIONS__GIT_CLONE__IMAGE - value: "{{ .Values.notebooks.gitClone.image.name }}:{{ .Values.notebooks.gitClone.image.tag }}" - - name: NB_ANONYMOUS_SESSIONS_ENABLED - value: {{ .Values.global.anonymousSessions.enabled | quote }} - - name: NB_SSH_ENABLED - value: {{ .Values.notebooks.ssh.enabled | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS - value: {{ .Values.notebooks.culling.idleSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS - value: {{ .Values.notebooks.culling.idleSecondsThreshold.anonymous | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__HIBERNATED_SECONDS - value: {{ .Values.notebooks.culling.hibernatedSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS - value: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS - value: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.anonymous | quote }} - - name: NB_AMALTHEA__GROUP - value: {{ .Values.amalthea.crdApiGroup }} - - name: NB_AMALTHEA__VERSION - value: {{ .Values.amalthea.crdApiVersion }} - - name: NB_AMALTHEA__PLURAL - value: {{ .Values.amalthea.crdNames.plural }} - - name: NB_AMALTHEA__CACHE_URL - value: http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher - name: NB_SENTRY__ENABLED value: {{ .Values.notebooks.sentry.enabled | quote }} - name: NB_SENTRY__DSN @@ -110,84 +50,14 @@ spec: value: {{ .Values.notebooks.sentry.sampleRate | quote }} - name: SENTRY_RELEASE value: {{ .Chart.Version | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED - value: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__DSN - value: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENV - value: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE - value: {{ .Values.notebooks.sessionSentry.gitClone.sampleRate | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED - value: {{ .Values.notebooks.sessionSentry.sidecar.enabled | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN - value: {{ .Values.notebooks.sessionSentry.sidecar.dsn | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV - value: {{ .Values.notebooks.sessionSentry.sidecar.environment | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE - value: {{ .Values.notebooks.sessionSentry.sidecar.sampleRate | quote }} - - name: NB_SESSIONS__CA_CERTS__IMAGE - value: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}" - - name: NB_SESSIONS__CA_CERTS__SECRETS - value: | - {{- .Values.global.certificates.customCAs | toYaml | nindent 16 }} - {{- with .Values.notebooks.sessionNodeSelector }} - - name: NB_SESSIONS__NODE_SELECTOR - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - {{- with .Values.notebooks.sessionAffinity }} - - name: NB_SESSIONS__AFFINITY - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - {{- with .Values.notebooks.sessionTolerations }} - - name: NB_SESSIONS__TOLERATIONS - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - name: KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace {{- include "certificates.env.python" . | nindent 12 }} - - name: NB_SESSIONS__ENFORCE_CPU_LIMITS - value: {{ .Values.notebooks.enforceCPULimits | quote }} - - name: NB_CLOUD_STORAGE__ENABLED - value: {{ .Values.notebooks.cloudstorage.enabled | quote }} - - name: NB_CLOUD_STORAGE__STORAGE_CLASS - value: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} - - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS - value: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} - - name: NB_VERSION - value: {{ .Values.notebooks.image.tag | quote }} - {{ if .Values.notebooks.sessionsNamespace }} - - name: NB_K8S__SESSIONS_NAMESPACE - value: {{ .Values.notebooks.sessionsNamespace | quote }} - {{ end }} - - name: NB_K8S__RENKU_NAMESPACE - value: {{ .Release.Namespace | quote }} - - name: NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_ID - value: renku - - name: NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_SECRET - value: {{ .Values.global.gateway.clientSecret | quote }} - - name: NB_KEYCLOAK_REALM - value: {{ include "renku.keycloak.realm" . | quote }} - - name: NB_SESSIONS__SSH__ENABLED - value: {{ .Values.notebooks.ssh.enabled | quote }} - {{- if not (kindIs "invalid" .Values.notebooks.ssh.hostKeySecret) }} - - name: NB_SESSIONS__SSH__HOST_KEY_SECRET - value: {{ .Values.notebooks.ssh.hostKeySecret | quote }} - {{- end }} - name: NB_DUMMY_STORES value: {{ .Values.notebooks.dummyStores | quote }} - - name: NB_DATA_SERVICE_URL - value: {{ printf "http://%s-data-service/api/data" .Release.Name}} - - name: NB_USER_SECRETS__SECRETS_STORAGE_SERVICE_URL - value: {{ printf "http://%s-secrets-storage" .Release.Name}} - - name: NB_USER_SECRETS__IMAGE - value: "{{ .Values.notebooks.secretsMount.image.repository}}:{{.Values.notebooks.secretsMount.image.tag}}" ports: - name: http containerPort: 8000 From 59e103681f2f6a87ab7788a4123015535b84c608 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 15 Jul 2024 17:43:52 +0200 Subject: [PATCH 03/42] squashme: minor helm fixes --- .../renku/templates/data-service/deployment.yaml | 11 ++++++----- helm-chart/renku/templates/notebooks/env-secret.yaml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index ed9abf3857..35fc912205 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -73,10 +73,6 @@ spec: value: /secrets/encryptionKey/encryptionKey - name: SECRETS_SERVICE_PUBLIC_KEY_PATH value: /secrets/publicKey/publicKey - - name: SERVER_DEFAULTS - value: /etc/renku-data-service/server_options/server_defaults.json - - name: SERVER_OPTIONS - value: /etc/renku-data-service/server_options/server_options.json - name: K8S_NAMESPACE value: {{ .Release.Namespace | quote }} - name: GITLAB_URL @@ -119,11 +115,16 @@ spec: value: {{ .Values.dataService.trustedProxies.proxiesCount | default "" | quote }} - name: REAL_IP_HEADER value: {{ .Values.dataService.trustedProxies.realIpHeader | default "" | quote }} + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace {{- include "certificates.env.python" . | nindent 12 }} {{- include "certificates.env.grpc" . | nindent 12 }} volumeMounts: - name: server-options - mountPath: /etc/renku-data-service/server_options + mountPath: /etc/renku-notebooks/server_options - mountPath: "/secrets/encryptionKey" name: encryption-key readOnly: true diff --git a/helm-chart/renku/templates/notebooks/env-secret.yaml b/helm-chart/renku/templates/notebooks/env-secret.yaml index 03bf4e2851..567e351198 100644 --- a/helm-chart/renku/templates/notebooks/env-secret.yaml +++ b/helm-chart/renku/templates/notebooks/env-secret.yaml @@ -41,6 +41,7 @@ stringData: NB_AMALTHEA__VERSION: {{ .Values.amalthea.crdApiVersion }} NB_AMALTHEA__PLURAL: {{ .Values.amalthea.crdNames.plural }} NB_AMALTHEA__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher" + NB_AMALTHEA_V2__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher" NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }} NB_SESSIONS__GIT_CLONE__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }} NB_SESSIONS__GIT_CLONE__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }} From 572695b96909622050aa85c3d98644f732f3ee3c Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 6 Aug 2024 14:04:50 +0200 Subject: [PATCH 04/42] chore: Update changelog title for 0.56.0 release --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6c919ea789..b294932b28 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,9 @@ .. _changelog: +0.56.0 +------ + + 0.55.0 ------ From 245b0b8aa11008cc683081fb794a4902407c1890 Mon Sep 17 00:00:00 2001 From: Lorenzo Cavazzi <43481553+lorenzo-cavazzi@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:46:18 +0200 Subject: [PATCH 05/42] refactor: revamp design for Renku 2.0 (#3726) New design for Renku 2.0 pages based on Bootstrap v5.3 components https://github.com/SwissDataScienceCenter/renku-ui/pull/3214 --- CHANGELOG.rst | 15 +++++++++++++++ helm-chart/renku/values.yaml | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b294932b28..3ff10a9171 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,21 @@ 0.56.0 ------ +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**✨ Improvements** + +- Revamp design for Renku 2.0 (`#3214 `__). + +**🐞 Bug Fixes** + +- Use standard HTML input fields for secret values (`#3233 `__). + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-ui 3.33.0 `_ 0.55.0 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index f4eb46133b..6e212e7298 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -672,7 +672,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.31.0" + tag: "3.33.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -861,7 +861,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.31.0" + tag: "3.33.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" From f692aa8950a770d91780c26e8a64b646fe535210 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 7 Aug 2024 13:02:58 +0200 Subject: [PATCH 06/42] feat: upgrade Amalthea to 0.12.2 (#3724) Co-authored-by: Alessandro Degano --- .github/workflows/pull-request-test.yml | 2 ++ CHANGELOG.rst | 18 ++++++++++++++++++ helm-chart/renku/requirements.yaml | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index 5e5030fbfb..f5fb0e8624 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -57,6 +57,7 @@ jobs: renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}} renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}} renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}} + amalthea: ${{ steps.deploy-comment.outputs.amalthea}} test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: @@ -97,6 +98,7 @@ jobs: renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}" renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}" renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}" + amalthea: "${{ needs.check-deploy.outputs.amalthea }}" extra_values: "${{ needs.check-deploy.outputs.extra-values }}" - name: Check existing renkubot comment if: needs.check-deploy.outputs.pr-contains-string == 'true' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3ff10a9171..5e606b19b1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,8 @@ 0.56.0 ------ +Renku ``0.56.0`` adds new features and improvements to several components. + User-Facing Changes ~~~~~~~~~~~~~~~~~~~ @@ -14,9 +16,25 @@ User-Facing Changes - Use standard HTML input fields for secret values (`#3233 `__). +Internal Changes +~~~~~~~~~~~~~~~~ + +**Improvements** + +- * **Amalthea**: Sessions can now run correctly on Kubernetes version 1.29. + +**🐞 Bug Fixes** + +- * **Amalthea**: Fix the repository for the scheduler image in the Amalthea Helm chart. +- * **Amalthea**: Properly load the namespace configuration when starting the operator. +- * **Amalthea**: Fix the missing health check endpoint for the old operator. + Individual Components ~~~~~~~~~~~~~~~~~~~~~ +- `amalthea 0.12.0 `_ +- `amalthea 0.12.1 `_ +- `amalthea 0.12.2 `_ - `renku-ui 3.33.0 `_ 0.55.0 diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 241fe30915..639efc5367 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,7 +23,7 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.11.0" + version: "0.12.2" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From 09d2e7df905403a14bd0b5fa94e0080b58cbdb7b Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 7 Aug 2024 15:55:52 +0200 Subject: [PATCH 07/42] feat!: add platform configuration (#3716) --- CHANGELOG.rst | 11 ++++++++++- helm-chart/renku/values.yaml | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5e606b19b1..7eb6ccff62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,12 @@ Renku ``0.56.0`` adds new features and improvements to several components. User-Facing Changes ~~~~~~~~~~~~~~~~~~~ +**🌟 New Features** + +- **UI**: Update incidents and maintenance banner and summary (`#3220 `__) +- **UI**: Add incidents and maintenance section in the admin panel (`#3220 `__) +- **Data Services**: Add platform configuration + **✨ Improvements** - Revamp design for Renku 2.0 (`#3214 `__). @@ -32,10 +38,13 @@ Internal Changes Individual Components ~~~~~~~~~~~~~~~~~~~~~ +- `renku-data-services 0.18.0 `_ +- `renku-data-services 0.18.1 `_ +- `renku-ui 3.32.0 `_ +- `renku-ui 3.33.0 `_ - `amalthea 0.12.0 `_ - `amalthea 0.12.1 `_ - `amalthea 0.12.2 `_ -- `renku-ui 3.33.0 `_ 0.55.0 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 6e212e7298..39991c607e 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1603,14 +1603,14 @@ platformInit: dataService: image: repository: renku/renku-data-service - tag: "0.17.0" + tag: "0.18.1" pullPolicy: IfNotPresent backgroundJobs: events: resources: {} image: repository: renku/data-service-background-jobs - tag: "0.17.0" + tag: "0.18.1" pullPolicy: IfNotPresent total: resources: {} @@ -1663,7 +1663,7 @@ authz: secretsStorage: image: repository: renku/secrets-storage - tag: "0.17.0" + tag: "0.18.1" pullPolicy: IfNotPresent service: type: ClusterIP From 57114d314ebaf399f312aee09ef030d118a4f8cd Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Mon, 12 Aug 2024 13:34:58 +0000 Subject: [PATCH 08/42] chore: create release 0.57.0 --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7eb6ccff62..6da49d45dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,10 @@ .. _changelog: +0.57.0 +------ + + + 0.56.0 ------ From 6e3a882b51ef1f3c51c4f938012ed0d3f1bbb5a0 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 12 Aug 2024 11:48:44 +0200 Subject: [PATCH 09/42] chore: add new amalthea helm chart as requirement --- helm-chart/renku/requirements.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 639efc5367..276ccc5cb2 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -24,6 +24,9 @@ dependencies: - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" version: "0.12.2" + - name: amalthea-sessions + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.0.9-new-operator-chart" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From 876bb99c38e2212362331c082c10de0cc914650b Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 12 Aug 2024 17:41:06 +0200 Subject: [PATCH 10/42] chore: bump amalthea dev version --- helm-chart/renku/requirements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 276ccc5cb2..3ba09ac572 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -26,7 +26,7 @@ dependencies: version: "0.12.2" - name: amalthea-sessions repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.0.9-new-operator-chart" + version: "0.0.10-new-operator-chart" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From 6536862a4b6d80ba75fbb6beebdfc5179d7e87ee Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 21 Aug 2024 10:41:45 +0200 Subject: [PATCH 11/42] chore: add rbac for amalthea sessions for notebooks --- .../renku/templates/data-service/rbac.yaml | 61 +++++++++++++++++++ .../renku/templates/notebooks/role.yaml | 12 ++++ 2 files changed, 73 insertions(+) diff --git a/helm-chart/renku/templates/data-service/rbac.yaml b/helm-chart/renku/templates/data-service/rbac.yaml index 6680e2fe2d..05f5c0c500 100644 --- a/helm-chart/renku/templates/data-service/rbac.yaml +++ b/helm-chart/renku/templates/data-service/rbac.yaml @@ -18,6 +18,67 @@ rules: - patch - delete - create + - apiGroups: + - "" + resources: + - pods + - pods/log + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - delete + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch + - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - delete + - patch + - apiGroups: + - {{ .Values.amalthea.crdApiGroup }} + resources: + - {{ .Values.amalthea.crdNames.plural }} + verbs: + - create + - update + - delete + - patch + - list + - get + - watch + - apiGroups: + - amalthea.dev + resources: + - amaltheasessions + verbs: + - create + - update + - delete + - patch + - list + - get + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/helm-chart/renku/templates/notebooks/role.yaml b/helm-chart/renku/templates/notebooks/role.yaml index 13ca5ad43d..1ebe6b008e 100644 --- a/helm-chart/renku/templates/notebooks/role.yaml +++ b/helm-chart/renku/templates/notebooks/role.yaml @@ -67,5 +67,17 @@ rules: - list - get - watch + - apiGroups: + - amalthea.dev + resources: + - amaltheasessions + verbs: + - create + - update + - delete + - patch + - list + - get + - watch {{ end }} {{ end }} From 09522c2be676e9480831f2a9aea1f33cef3754ee Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Fri, 23 Aug 2024 10:58:02 +0200 Subject: [PATCH 12/42] chore: use the new action to deploy the new amalthea --- .github/workflows/pull-request-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index f5fb0e8624..ca0ccccd6e 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -58,12 +58,13 @@ jobs: renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}} renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}} amalthea: ${{ steps.deploy-comment.outputs.amalthea}} + amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}} test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: - uses: actions/checkout@v4.1.7 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@feat-add-new-amalthea-helm-chart with: string: /deploy pr_ref: ${{ github.event.number }} @@ -79,7 +80,7 @@ jobs: - uses: actions/checkout@v4.1.7 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@feat-add-new-amalthea-helm-chart env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -99,6 +100,7 @@ jobs: renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}" renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}" amalthea: "${{ needs.check-deploy.outputs.amalthea }}" + amalthea_sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}" extra_values: "${{ needs.check-deploy.outputs.extra-values }}" - name: Check existing renkubot comment if: needs.check-deploy.outputs.pr-contains-string == 'true' From ebedb036d40bde8fb87943d8583a610977069bb4 Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Mon, 12 Aug 2024 13:34:58 +0000 Subject: [PATCH 13/42] chore: create release 0.57.0 --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e996222260..9b9345aa31 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,10 @@ .. _changelog: +0.57.0 +------ + + + 0.56.2 ------ From 35746c788f554a1f98913cbad224d76640f87896 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Fri, 23 Aug 2024 15:37:03 +0200 Subject: [PATCH 14/42] chore: upadte renku-actions version to 1.12.0 --- .github/workflows/cron-jobs.yaml | 2 +- .github/workflows/publish-helm-chart.yml | 2 +- .github/workflows/publish-master-merges.yaml | 2 +- .github/workflows/pull-request-test.yml | 10 +++++----- .github/workflows/renku-dev-test.yaml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index b4194ec918..933ebf8d51 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.0 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 30422141e1..74a3d03057 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -15,7 +15,7 @@ jobs: - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.11.3 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.0 env: CHART_DIR: helm-chart/ CHART_NAME: renku diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index df49c7c868..70ca92393d 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.11.3 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.0 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index ca0ccccd6e..ff8a3cbab4 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -64,7 +64,7 @@ jobs: steps: - uses: actions/checkout@v4.1.7 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@feat-add-new-amalthea-helm-chart + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.0 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v4.1.7 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@feat-add-new-amalthea-helm-chart + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.0 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -124,7 +124,7 @@ jobs: needs: [check-deploy, deploy-pr] runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -152,7 +152,7 @@ jobs: ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.0 if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -185,7 +185,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.0 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index 60074e2941..8e90df5512 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku From 3b28476462e39e55e2913af37f266c8c6ea2efc4 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Fri, 23 Aug 2024 23:13:44 +0200 Subject: [PATCH 15/42] chore: update actions to 1.12.1 --- .github/workflows/cron-jobs.yaml | 2 +- .github/workflows/publish-helm-chart.yml | 2 +- .github/workflows/publish-master-merges.yaml | 2 +- .github/workflows/pull-request-test.yml | 10 +++++----- .github/workflows/renku-dev-test.yaml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index 933ebf8d51..c38fd94f8f 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.0 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 74a3d03057..d3fdcaabd2 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -15,7 +15,7 @@ jobs: - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.0 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.1 env: CHART_DIR: helm-chart/ CHART_NAME: renku diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index 70ca92393d..22d8daf8ca 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.0 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.1 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index ff8a3cbab4..0f4592fe2d 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -64,7 +64,7 @@ jobs: steps: - uses: actions/checkout@v4.1.7 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.0 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v4.1.7 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.0 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -124,7 +124,7 @@ jobs: needs: [check-deploy, deploy-pr] runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.0 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -152,7 +152,7 @@ jobs: ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.0 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1 if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -185,7 +185,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.0 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index 8e90df5512..1a1f9a123f 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.0 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku From f62d9644cf39c05ca4a839d2fcaef54f23aa5c35 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Fri, 23 Aug 2024 23:13:58 +0200 Subject: [PATCH 16/42] squashme: minor helm change --- helm-chart/renku/templates/data-service/deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index 35fc912205..e657cd9b4f 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -120,6 +120,11 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace + - name: SESSIONS_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace {{- include "certificates.env.python" . | nindent 12 }} {{- include "certificates.env.grpc" . | nindent 12 }} volumeMounts: From 129790243333b5c8c0417b4417a831890fc8b30c Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Sat, 24 Aug 2024 13:00:02 +0200 Subject: [PATCH 17/42] squashme: add keycloak oidc issuer url --- helm-chart/renku/templates/_helpers.tpl | 4 ++++ helm-chart/renku/templates/notebooks/env-secret.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl index 7cec86a39f..852f77ebaf 100644 --- a/helm-chart/renku/templates/_helpers.tpl +++ b/helm-chart/renku/templates/_helpers.tpl @@ -149,6 +149,10 @@ KC_DB_PASSWORD: {{ default (randAlphaNum 64) .Values.global.keycloak.postgresPas {{- end -}} {{- end -}} +{{- define "renku.keycloakIssuerUrl" -}} +{{- printf "%s/realms/%s" (include "renku.keycloakUrl" . | trimSuffix "/") (include "renku.keycloak.realm" .) -}} +{{- end -}} + {{/* Common labels */}} diff --git a/helm-chart/renku/templates/notebooks/env-secret.yaml b/helm-chart/renku/templates/notebooks/env-secret.yaml index 567e351198..c229aab657 100644 --- a/helm-chart/renku/templates/notebooks/env-secret.yaml +++ b/helm-chart/renku/templates/notebooks/env-secret.yaml @@ -20,6 +20,7 @@ stringData: NB_SESSIONS__OIDC__CLIENT_SECRET: {{ .Values.notebooks.oidc.clientSecret }} NB_SESSIONS__OIDC__AUTH_URL: {{ .Values.notebooks.oidc.authUrl }} NB_SESSIONS__OIDC__TOKEN_URL: {{ .Values.notebooks.oidc.tokenUrl }} + NB_SESSIONS__OIDC__ISSUER_URL: {{ template "renku.keycloakIssuerUrl" . }} NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }} NB_SESSIONS__INGRESS__HOST: {{ .Values.notebooks.sessionIngress.host }} NB_SESSIONS__INGRESS__TLS_SECRET: {{ .Values.notebooks.sessionIngress.tlsSecret }} From 0926488c2b242cfcc09bff837b7c4740d51448f7 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 4 Sep 2024 14:17:16 +0200 Subject: [PATCH 18/42] squashme: upgrade to renku-actions 1.12.2 --- .github/workflows/cron-jobs.yaml | 2 +- .github/workflows/publish-helm-chart.yml | 2 +- .github/workflows/publish-master-merges.yaml | 2 +- .github/workflows/pull-request-test.yml | 10 +++++----- .github/workflows/renku-dev-test.yaml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index c38fd94f8f..cb36389683 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.2 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index d3fdcaabd2..0ec7f143d6 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -15,7 +15,7 @@ jobs: - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.2 env: CHART_DIR: helm-chart/ CHART_NAME: renku diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index 22d8daf8ca..c5ef17de71 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.12.2 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index 0f4592fe2d..5445f47a44 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -64,7 +64,7 @@ jobs: steps: - uses: actions/checkout@v4.1.7 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.2 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v4.1.7 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.2 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -124,7 +124,7 @@ jobs: needs: [check-deploy, deploy-pr] runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.2 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -152,7 +152,7 @@ jobs: ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.2 if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -185,7 +185,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.2 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index 1a1f9a123f..54f31f25bf 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.2 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku From cd54c4f4c24d4b23d2be61c95ed3692f0ab3df3f Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Wed, 9 Oct 2024 14:18:32 +0000 Subject: [PATCH 19/42] chore: create release 0.60.0 --- CHANGELOG.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5e8eac694c..2ca04ad1de 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,15 @@ .. _changelog: +0.60.0 +------ + + + +0.59.0 +------ + + + 0.58.1 ------ @@ -83,7 +93,7 @@ Internal Changes Individual Components ~~~~~~~~~~~~~~~~~~~~~ -- `renku-search 0.6.1 `_ +- `renku-search 0.6.1 `_ - `renku-ui 3.36.0 `_ - `renku-ui 3.37.0 `_ - `renku-ui 3.37.1 `_ From 5aa125115ce450f9c92b60644b1e1aaebbbe9afe Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:08:58 +0200 Subject: [PATCH 20/42] chore: Mute selenium dependency updates (#3574) --- .scala-steward.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.scala-steward.conf b/.scala-steward.conf index 0d337b2c64..14bb7f406d 100644 --- a/.scala-steward.conf +++ b/.scala-steward.conf @@ -7,3 +7,6 @@ # directory itself also contains a build.sbt the dot can be used to # specify it. Default: ["."] buildRoots = [ "acceptance-tests" ] +updates.ignore = [ + { groupId = "org.seleniumhq.selenium", artifactId = "selenium-java" } +] From 065285d9e895a4f0450511503fe7a770de7cd666 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 4 Sep 2024 11:42:50 +0200 Subject: [PATCH 21/42] feat!: gateway refactor (#3721) The previous gateway components are replaced with a single API Gateway responsible for login and reverse proxying requests. Details: * Remove the `gateway-auth` Python component * Update the `gateway-revproxy` Go component (now `gateway`) * Update the `ui-server` to remove login responsibilities --------- Co-authored-by: Tasko Olevski --- helm-chart/renku/templates/gateway/configmap.yaml | 2 -- helm-chart/renku/values.yaml | 10 ---------- helm-chart/values.yaml.changelog.md | 2 ++ 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/helm-chart/renku/templates/gateway/configmap.yaml b/helm-chart/renku/templates/gateway/configmap.yaml index c163177252..f426e970ca 100644 --- a/helm-chart/renku/templates/gateway/configmap.yaml +++ b/helm-chart/renku/templates/gateway/configmap.yaml @@ -69,8 +69,6 @@ data: scopes: ["openid", "api", "read_user", "read_repository"] callbackURI: {{ printf "%s/api/auth/callback" (include "renku.baseUrl" .) }} usePKCE: false - oldGitLabLogout: {{ .Values.gateway.oldGitLabLogout | default false }} - logoutGitLabUponRenkuLogout: {{ .Values.gateway.logoutGitLabUponRenkuLogout | default true }} redis: type: redis addresses: diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 8a75fb54b8..491bdee4b5 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1288,16 +1288,6 @@ gateway: # cpu: 100m # memory: 128Mi - # GitLab has introduced a new logout behavior in 12.7.0 - # which was initially broken and fixed in 12.9.0. - # Set this to 'true' for versions < 12.7.0, leave it to - # 'false' for versions >= 12.9.0. - oldGitLabLogout: false - # For deployments with an external GitLab instance, we don't want to terminate - # a GitLab session when users log out from Renku. By default, we assume that a - # dedicated GitLab instance is deployed, so, logging out from Renku should - # trigger a logout from GitLab. - logoutGitLabUponRenkuLogout: true # sentry configuration sentry: enabled: false diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index bb7064242f..db0acc3287 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -8,6 +8,8 @@ Please follow this convention when adding a new row ## Upgrading to Renku 0.57.0 * DELETE ``gateway.image.auth`` has been removed. +* DELETE ``gateway.oldGitLabLogout`` has been removed. +* DELETE ``gateway.logoutGitLabUponRenkuLogout`` has been removed. * EDIT ``gateway.reverseProxy`` settings have been moved to ``gateway``: Old From 4dafd22574e602f1ecda27f55116f1bcedd3a008 Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:08:24 +0200 Subject: [PATCH 22/42] feat: Update search services to 0.5.0 (#3754) --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2ca04ad1de..b440323bf0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -57,6 +57,10 @@ Renku 2.0 dashboard displayed content not related to you. User-Facing Changes ~~~~~~~~~~~~~~~~~~~ +**🌟 New Features** + +- **Search Services**: Enable searching by prefix of indexed words + **✨ Improvements** - **UI**: Polish Renku 2.0 pages and elements according to the latest design changes (`#3254 `__). From 764e1000708a8e1899e004fa6f87727dd049c6a6 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 12 Sep 2024 14:42:26 +0200 Subject: [PATCH 23/42] fix: implement browser logout from the gateway (#3764) Add browser-based logout for Keycloak and Gitlab. Previously, the logout flow would not end the Keycloak nor the GitLab browser session. This prevented users from switching accounts because their existing Keycloak and GitLab sessions would be re-used when logging in with the renku gateway. --- cypress-tests/cypress/e2e/useSession.cy.ts | 2 ++ helm-chart/renku/templates/gateway/configmap.yaml | 2 ++ helm-chart/renku/values.yaml | 10 ++++++++++ helm-chart/values.yaml.changelog.md | 2 -- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cypress-tests/cypress/e2e/useSession.cy.ts b/cypress-tests/cypress/e2e/useSession.cy.ts index 22b22d712c..c15fef18fa 100644 --- a/cypress-tests/cypress/e2e/useSession.cy.ts +++ b/cypress-tests/cypress/e2e/useSession.cy.ts @@ -189,6 +189,8 @@ describe("Basic public project functionality", () => { // Stop the session -- mind that anonymous users cannot pause sessions cy.deleteSession({ fromSessionPage: true }); + + cy.robustLogin(); }); it("Start a new session on a project without permissions.", () => { diff --git a/helm-chart/renku/templates/gateway/configmap.yaml b/helm-chart/renku/templates/gateway/configmap.yaml index f426e970ca..c163177252 100644 --- a/helm-chart/renku/templates/gateway/configmap.yaml +++ b/helm-chart/renku/templates/gateway/configmap.yaml @@ -69,6 +69,8 @@ data: scopes: ["openid", "api", "read_user", "read_repository"] callbackURI: {{ printf "%s/api/auth/callback" (include "renku.baseUrl" .) }} usePKCE: false + oldGitLabLogout: {{ .Values.gateway.oldGitLabLogout | default false }} + logoutGitLabUponRenkuLogout: {{ .Values.gateway.logoutGitLabUponRenkuLogout | default true }} redis: type: redis addresses: diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 491bdee4b5..8a75fb54b8 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1288,6 +1288,16 @@ gateway: # cpu: 100m # memory: 128Mi + # GitLab has introduced a new logout behavior in 12.7.0 + # which was initially broken and fixed in 12.9.0. + # Set this to 'true' for versions < 12.7.0, leave it to + # 'false' for versions >= 12.9.0. + oldGitLabLogout: false + # For deployments with an external GitLab instance, we don't want to terminate + # a GitLab session when users log out from Renku. By default, we assume that a + # dedicated GitLab instance is deployed, so, logging out from Renku should + # trigger a logout from GitLab. + logoutGitLabUponRenkuLogout: true # sentry configuration sentry: enabled: false diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index db0acc3287..bb7064242f 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -8,8 +8,6 @@ Please follow this convention when adding a new row ## Upgrading to Renku 0.57.0 * DELETE ``gateway.image.auth`` has been removed. -* DELETE ``gateway.oldGitLabLogout`` has been removed. -* DELETE ``gateway.logoutGitLabUponRenkuLogout`` has been removed. * EDIT ``gateway.reverseProxy`` settings have been moved to ``gateway``: Old From aa44aa2531951a4e1172357b845f4943d4ee2027 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 13 Sep 2024 10:42:33 +0200 Subject: [PATCH 24/42] feat: update renku-data-services to v0.20.0 and renku-ui to 3.35.0 (#3765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * data svc release v0.20.0 * bump UI -> 3.35.0 * update changelog * Update CHANGELOG.rst Co-authored-by: Laura * apply CHANGELOG suggestions --------- Co-authored-by: Flora Thiebaut Co-authored-by: Rok Roškar Co-authored-by: Laura Co-authored-by: Andrea Cordoba --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b440323bf0..8bb84f3901 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -60,6 +60,8 @@ User-Facing Changes **🌟 New Features** - **Search Services**: Enable searching by prefix of indexed words +- **UI**: Support saving and managing credentials for Renku 2.0 data sources (`#3266 `__). +- **UI**: Add members to groups and projects in Renku 2.0 by username instead of email (`#3270 `__). **✨ Improvements** @@ -194,6 +196,7 @@ User-Facing Changes - **UI**: Show the status of a session via a dynamic browser tab icon (`#3249 `__). - **UI**: Display session details in session page in Renku 2.0 (`#3258 `__) - **UI**: Set default namespace when creating a new Renku 2.0 project (`#3264 `__). +- **Data Services**: Do not show user emails and use usernames instead for all interactions **🐞 Bug Fixes** From d38084a7b3674e715ba98edd6b328e5a7cdd9e06 Mon Sep 17 00:00:00 2001 From: Laura Date: Mon, 16 Sep 2024 10:56:05 +0200 Subject: [PATCH 25/42] chore: polish release 0.57.0 changelog --- CHANGELOG.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8bb84f3901..91fb611565 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -59,9 +59,7 @@ User-Facing Changes **🌟 New Features** -- **Search Services**: Enable searching by prefix of indexed words - **UI**: Support saving and managing credentials for Renku 2.0 data sources (`#3266 `__). -- **UI**: Add members to groups and projects in Renku 2.0 by username instead of email (`#3270 `__). **✨ Improvements** @@ -196,7 +194,6 @@ User-Facing Changes - **UI**: Show the status of a session via a dynamic browser tab icon (`#3249 `__). - **UI**: Display session details in session page in Renku 2.0 (`#3258 `__) - **UI**: Set default namespace when creating a new Renku 2.0 project (`#3264 `__). -- **Data Services**: Do not show user emails and use usernames instead for all interactions **🐞 Bug Fixes** From d1148faefb24dbe27193ec1a654c6c6c9be24402 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 17 Sep 2024 17:15:00 +0200 Subject: [PATCH 26/42] squashme: add amalthea session cache config --- helm-chart/renku/templates/notebooks/deployment.yaml | 6 ++++++ helm-chart/renku/templates/notebooks/network-policy.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/helm-chart/renku/templates/notebooks/deployment.yaml b/helm-chart/renku/templates/notebooks/deployment.yaml index 8e65a6d313..978d466f12 100644 --- a/helm-chart/renku/templates/notebooks/deployment.yaml +++ b/helm-chart/renku/templates/notebooks/deployment.yaml @@ -43,6 +43,12 @@ spec: value: {{ .Values.amalthea.crdApiVersion }} - name: K8S_WATCHER_CR_PLURAL value: {{ .Values.amalthea.crdNames.plural }} + - name: K8S_WATCHER_AMALTHEA_SESSION_GROUP + value: amalthea.dev + - name: K8S_WATCHER_AMALTHEA_SESSION_VERSION + value: v1alpha1 + - name: K8S_WATCHER_AMALTHEA_SESSION_PLURAL + value: amaltheasessions - name: K8S_WATCHER_NAMESPACES {{ if .Values.notebooks.sessionsNamespace }} value: {{ list .Release.Namespace .Values.notebooks.sessionsNamespace | uniq | toJson | quote }} diff --git a/helm-chart/renku/templates/notebooks/network-policy.yaml b/helm-chart/renku/templates/notebooks/network-policy.yaml index 7de4810309..7f954dad17 100644 --- a/helm-chart/renku/templates/notebooks/network-policy.yaml +++ b/helm-chart/renku/templates/notebooks/network-policy.yaml @@ -15,6 +15,10 @@ spec: matchLabels: app: {{ template "renku.notebooks.name" . }} release: {{ .Release.Name }} + - podSelector: + matchLabels: + app: renku-data-service + release: {{ .Release.Name }} ports: - protocol: TCP port: http From c042e81c7e2049b8dd657bf72c9a0dd2b27acf4a Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 23 Oct 2024 17:47:18 +0200 Subject: [PATCH 27/42] feat: upgrade amalthea to 0.13.0 --- CHANGELOG.rst | 4 ++++ helm-chart/renku/requirements.yaml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 91fb611565..288707cff8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ ------ +Individual components +~~~~~~~~~~~~~~~~~~~~~~ + +- `amalthea 0.13.0 `_ 0.59.0 ------ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index b83c2ba5d1..a77efa6c98 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,10 +23,10 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.12.3" + version: "0.13.0" - name: amalthea-sessions repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.0.10-new-operator-chart" + version: "0.13.0" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From 5669fb12b5eacb39e1ce9c40abbc4ba57c67659a Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Wed, 9 Oct 2024 14:18:32 +0000 Subject: [PATCH 28/42] chore: create release 0.60.0 --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da523aa781..ec729d7a36 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,9 @@ .. _changelog: +0.60.0 +------ + + 0.59.2 ------ From ee8bbebd37c2c176a045ac6338fbdb3c723f0174 Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:08:58 +0200 Subject: [PATCH 29/42] chore: Mute selenium dependency updates (#3574) --- .scala-steward.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.scala-steward.conf b/.scala-steward.conf index 0d337b2c64..14bb7f406d 100644 --- a/.scala-steward.conf +++ b/.scala-steward.conf @@ -7,3 +7,6 @@ # directory itself also contains a build.sbt the dot can be used to # specify it. Default: ["."] buildRoots = [ "acceptance-tests" ] +updates.ignore = [ + { groupId = "org.seleniumhq.selenium", artifactId = "selenium-java" } +] From 34d7cf8b8f6e8dfded2831540bf4c5a4683cce21 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 28 Oct 2024 16:04:36 +0100 Subject: [PATCH 30/42] feat: allow session TTL values to be set (#3826) --- helm-chart/renku/templates/gateway/configmap.yaml | 6 ++++++ helm-chart/renku/values.yaml | 9 +++++++++ helm-chart/values.yaml.changelog.md | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/helm-chart/renku/templates/gateway/configmap.yaml b/helm-chart/renku/templates/gateway/configmap.yaml index c163177252..ab00288a2f 100644 --- a/helm-chart/renku/templates/gateway/configmap.yaml +++ b/helm-chart/renku/templates/gateway/configmap.yaml @@ -21,6 +21,12 @@ data: {{- toYaml . | nindent 8 }} {{- end }} sessions: + {{- if .Values.gateway.idleSessionTTLSeconds }} + idleSessionTTLSeconds: {{ .Values.gateway.idleSessionTTLSeconds }} + {{- end }} + {{- if .Values.gateway.maxSessionTTLSeconds }} + maxSessionTTLSeconds: {{ .Values.gateway.maxSessionTTLSeconds }} + {{- end }} authorizationVerifiers: - issuer: {{ printf "%s/realms/%s" (include "renku.keycloakUrl" . | trimSuffix "/") (include "renku.keycloak.realm" .) }} audience: renku diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index a7dc199d25..015234edf9 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1266,6 +1266,15 @@ gateway: ## average rate units are requests per second average: 20 burst: 100 + ## How long a session will be kept alive if it is idle. + ## If set to 0 then there is no TTL. + ## If not set, defaults to 14400 (4 hours). + idleSessionTTLSeconds: + ## The maximum duration a session can be kept alive. + ## If set to 0 then there is no TTL. + ## If this value is set to a value other than 0 then it must be greater than idleSessionTTLSeconds. + ## If not set, defaults to 86400 (24 hours). + maxSessionTTLSeconds: ## For production deployment, you will need to define the secret key. ## This is a random string, used for cryptographic operations on cookies and sensitive information. ## Use `openssl rand -hex 32`. diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index 756465bed9..f629dbb9b3 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,11 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.60.0 + +* NEW ``gateway.idleSessionTTLSeconds`` to set the session idle TTL in seconds. +* NEW ``gateway.maxSessionTTLSeconds`` to set the session max TTL in seconds. + ## Upgrading to Renku 0.59.1 * NEW ``notebooks.bypassCacheOnFailure`` has been added. Setting this to false prevents renku-notebooks to call From e78765ba4d386730604973c782aadc254e425841 Mon Sep 17 00:00:00 2001 From: Lorenzo Cavazzi <43481553+lorenzo-cavazzi@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:52:15 +0100 Subject: [PATCH 31/42] feat: improve UX and better support GitHub connections (#3830) --- CHANGELOG.rst | 26 ++++++++++++++++++++++++++ helm-chart/renku/values.yaml | 10 +++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec729d7a36..17b3dfa22d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,32 @@ 0.60.0 ------ +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**✨ Improvements** + +- **UI & Data services**: Improve UX for users connecting to GitHub (`#3332 `__). +- **UI**: Update project and group membership settings (`#3374 `__). +- **UI**: Improve UX for adding data connectors (`#3368 `__). + +**🐞 Bug Fixes** + +- **UI**: Avoid unexpected redirects when clicking on the 2.0 dashboard session buttons (`#3378 `__). + +Internal Changes +~~~~~~~~~~~~~~~~ + +**Improvements** + +- **UI**: Update the Admin page to set up Connected services (`#3332 `__). + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-data-services 0.25.0 `__ +- `renku-ui 3.41.0 `_ + 0.59.2 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 015234edf9..709203b011 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -654,7 +654,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.40.1" + tag: "3.41.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -843,7 +843,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.40.1" + tag: "3.41.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" @@ -1588,14 +1588,14 @@ dataService: create: true image: repository: renku/renku-data-service - tag: "0.24.2" + tag: "0.25.0" pullPolicy: IfNotPresent backgroundJobs: events: resources: {} image: repository: renku/data-service-background-jobs - tag: "0.24.2" + tag: "0.25.0" pullPolicy: IfNotPresent total: resources: {} @@ -1648,7 +1648,7 @@ authz: secretsStorage: image: repository: renku/secrets-storage - tag: "0.24.2" + tag: "0.25.0" pullPolicy: IfNotPresent service: type: ClusterIP From 91acad84469f27e004e6c42d7fee4592164419ea Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 29 Oct 2024 10:09:11 -0400 Subject: [PATCH 32/42] fix: updgrade csi rclone to 0.3.5 (#3831) --- CHANGELOG.rst | 5 +++++ helm-chart/renku/requirements.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 17b3dfa22d..f6460c699a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,11 +55,16 @@ Internal Changes **Bug Fixes** - **Data services**: Handle spaces in ``provider_id`` for connected services (`#482 `__). +- **csi-rclone**: Do not log potentially sensitive data in error messages. +- **csi-rclone**: Properly handle encrypted secrets with the new annotation-based storage class. + Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-data-services 0.24.2 `__ +- `csi-rclone 0.3.4 `__ +- `csi-rclone 0.3.5 `__ 0.59.1 ------ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 7a214673d5..c1c773a880 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -30,7 +30,7 @@ dependencies: condition: notebooks.cloudstorage.s3.installDatashim - name: csi-rclone repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.3.3" + version: "0.3.5" condition: global.csi-rclone.install - name: solr repository: "oci://registry-1.docker.io/bitnamicharts" From 9a949bd29d04b28b1d0b6e901b9f5975c1e2aabe Mon Sep 17 00:00:00 2001 From: Andrea Cordoba Date: Wed, 30 Oct 2024 10:40:58 +0100 Subject: [PATCH 33/42] fix: update the getProjectByNamespaceAPIV2 function in the get project endpoint for e2e tests --- cypress-tests/cypress/support/utils/projectsV2.utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress-tests/cypress/support/utils/projectsV2.utils.ts b/cypress-tests/cypress/support/utils/projectsV2.utils.ts index 1770aaff91..e55665f105 100644 --- a/cypress-tests/cypress/support/utils/projectsV2.utils.ts +++ b/cypress-tests/cypress/support/utils/projectsV2.utils.ts @@ -23,7 +23,7 @@ export function getUserNamespaceAPIV2(): Cypress.Chainable { /** Get a project by using only the API. */ export function getProjectByNamespaceAPIV2(newProjectProps: ProjectIdentifierV2): Cypress.Chainable { - return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/projects/${newProjectProps.namespace}/${newProjectProps.slug}` }); + return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/namespaces/${newProjectProps.namespace}/projects/${newProjectProps.slug}` }); } /** Create a project (if the project is missing) by using only the API. */ From ee34edf7403cb16fd046670a03c524a27c7f685f Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Thu, 7 Nov 2024 00:07:01 -0500 Subject: [PATCH 34/42] feat: add ulid generation function to postgres --- scripts/init-db/Dockerfile | 1 + scripts/init-db/generate_ulid_func.sql | 83 ++++++++++++++++++++++++++ scripts/init-db/renku_db_init.py | 5 +- scripts/init-db/utils.py | 15 +++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 scripts/init-db/generate_ulid_func.sql diff --git a/scripts/init-db/Dockerfile b/scripts/init-db/Dockerfile index fb2b22deb4..e194593dd5 100644 --- a/scripts/init-db/Dockerfile +++ b/scripts/init-db/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.11-slim-bullseye RUN apt-get update && apt-get install -y \ + postgresql-client \ tini && \ rm -rf /var/lib/apt/lists/* COPY . . diff --git a/scripts/init-db/generate_ulid_func.sql b/scripts/init-db/generate_ulid_func.sql new file mode 100644 index 0000000000..90b2aa8305 --- /dev/null +++ b/scripts/init-db/generate_ulid_func.sql @@ -0,0 +1,83 @@ +-- From https://github.com/geckoboard/pgulid/blob/master/pgulid.sql +-- Taken at commit sha b265253 +-- pgulid is based on OK Log's Go implementation of the ULID spec +-- +-- https://github.com/oklog/ulid +-- https://github.com/ulid/spec +-- +-- Copyright 2016 The Oklog Authors +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- NOTE: REPLACE will error if you change the name, args or return type of the function +-- There is no CREATE IF EXISTS, this is the closest thing that gives similar functionality +CREATE OR REPLACE FUNCTION generate_ulid() +RETURNS TEXT +AS $$ +DECLARE + -- Crockford's Base32 + encoding BYTEA = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + timestamp BYTEA = E'\\000\\000\\000\\000\\000\\000'; + output TEXT = ''; + + unix_time BIGINT; + ulid BYTEA; +BEGIN + -- 6 timestamp bytes + unix_time = (EXTRACT(EPOCH FROM CLOCK_TIMESTAMP()) * 1000)::BIGINT; + timestamp = SET_BYTE(timestamp, 0, (unix_time >> 40)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 1, (unix_time >> 32)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 2, (unix_time >> 24)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 3, (unix_time >> 16)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 4, (unix_time >> 8)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 5, unix_time::BIT(8)::INTEGER); + + -- 10 entropy bytes + ulid = timestamp || gen_random_bytes(10); + + -- Encode the timestamp + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 224) >> 5)); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 31))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 1) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 1) & 7) << 2) | ((GET_BYTE(ulid, 2) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 2) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 2) & 1) << 4) | ((GET_BYTE(ulid, 3) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 3) & 15) << 1) | ((GET_BYTE(ulid, 4) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 4) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 4) & 3) << 3) | ((GET_BYTE(ulid, 5) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 5) & 31))); + + -- Encode the entropy + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 6) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 6) & 7) << 2) | ((GET_BYTE(ulid, 7) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 7) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 7) & 1) << 4) | ((GET_BYTE(ulid, 8) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 8) & 15) << 1) | ((GET_BYTE(ulid, 9) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 9) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 9) & 3) << 3) | ((GET_BYTE(ulid, 10) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 10) & 31))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 11) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 11) & 7) << 2) | ((GET_BYTE(ulid, 12) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 12) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 12) & 1) << 4) | ((GET_BYTE(ulid, 13) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 13) & 15) << 1) | ((GET_BYTE(ulid, 14) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 14) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 14) & 3) << 3) | ((GET_BYTE(ulid, 15) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 15) & 31))); + + RETURN output; +END +$$ +LANGUAGE plpgsql +VOLATILE; diff --git a/scripts/init-db/renku_db_init.py b/scripts/init-db/renku_db_init.py index e4789b38e0..2b33b2fb43 100644 --- a/scripts/init-db/renku_db_init.py +++ b/scripts/init-db/renku_db_init.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from queries import DatabaseInit -from utils import get_db_connection +from utils import create_ulid_func, get_db_connection logging.basicConfig(level=logging.INFO) @@ -132,7 +132,7 @@ def main(): config.renku_db_name, config.renku_db_password, postgres_db_connection, - ["pg_trgm"], + ["pg_trgm", "pgcrypto"], config.db_admin_username, ) db_init.create_database() @@ -147,6 +147,7 @@ def main(): renku_conn.set_session(autocommit=True) db_init.set_connection(renku_conn) db_init.set_extensions_and_roles() + create_ulid_func(config.db_admin_username, config.db_admin_password, config.renku_db_name) if __name__ == "__main__": diff --git a/scripts/init-db/utils.py b/scripts/init-db/utils.py index 10355b8877..257390821b 100644 --- a/scripts/init-db/utils.py +++ b/scripts/init-db/utils.py @@ -1,4 +1,5 @@ import logging +from subprocess import STDOUT, check_output import requests @@ -36,3 +37,17 @@ def gitlab_is_online(url: str) -> int: f"Gitlab is not available at {url}, status code is {res.status_code}" ) return res.status_code + + +def create_ulid_func(username: str, password: str, db_name: str): + try: + output = check_output( + ["psql", "-U", username, "-d", db_name, "-f", "generate_ulid_func.sql"], + stderr=STDOUT, + env={"PGPASSWORD": password}, + ).decode("utf-8") + except Exception: + logging.error(f"Failed to create the ulid generation function:\n{output}") + raise + else: + logging.info(f"Created the ulid generation function:\n{output}") From c1315f8a03fea928a84075237895fcc6a2c4ac8b Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Thu, 7 Nov 2024 00:14:59 -0500 Subject: [PATCH 35/42] chore: remove outdated changelog --- CHANGELOG.rst | 25 ------------------------- scripts/init-db/renku_db_init.py | 2 +- scripts/init-db/utils.py | 19 +++++++------------ 3 files changed, 8 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b6bae85c3..1d60874a6c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,32 +3,7 @@ 0.61.0 ------ -User-Facing Changes -~~~~~~~~~~~~~~~~~~~ - -**✨ Improvements** - -- **UI & Data services**: Improve UX for users connecting to GitHub (`#3332 `__). -- **UI**: Update project and group membership settings (`#3374 `__). -- **UI**: Improve UX for adding data connectors (`#3368 `__). - -**🐞 Bug Fixes** - -- **UI**: Avoid unexpected redirects when clicking on the 2.0 dashboard session buttons (`#3378 `__). - -Internal Changes -~~~~~~~~~~~~~~~~ - -**Improvements** - -- **UI**: Update the Admin page to set up Connected services (`#3332 `__). -Individual Components -~~~~~~~~~~~~~~~~~~~~~ - -- `renku-data-services 0.25.0 `__ -- `renku-ui 3.41.0 `_ -- `amalthea 0.13.0 `_ 0.60.0 ------ diff --git a/scripts/init-db/renku_db_init.py b/scripts/init-db/renku_db_init.py index 2b33b2fb43..0e2bfc31c0 100644 --- a/scripts/init-db/renku_db_init.py +++ b/scripts/init-db/renku_db_init.py @@ -147,7 +147,7 @@ def main(): renku_conn.set_session(autocommit=True) db_init.set_connection(renku_conn) db_init.set_extensions_and_roles() - create_ulid_func(config.db_admin_username, config.db_admin_password, config.renku_db_name) + create_ulid_func(config.db_admin_username, config.db_admin_password, config.renku_db_name, config.db_host, config.db_port) if __name__ == "__main__": diff --git a/scripts/init-db/utils.py b/scripts/init-db/utils.py index 257390821b..ffb09e8e5f 100644 --- a/scripts/init-db/utils.py +++ b/scripts/init-db/utils.py @@ -39,15 +39,10 @@ def gitlab_is_online(url: str) -> int: return res.status_code -def create_ulid_func(username: str, password: str, db_name: str): - try: - output = check_output( - ["psql", "-U", username, "-d", db_name, "-f", "generate_ulid_func.sql"], - stderr=STDOUT, - env={"PGPASSWORD": password}, - ).decode("utf-8") - except Exception: - logging.error(f"Failed to create the ulid generation function:\n{output}") - raise - else: - logging.info(f"Created the ulid generation function:\n{output}") +def create_ulid_func(username: str, password: str, db_name: str, host: str, port: int): + output = check_output( + ["psql", "-U", username, "-d", db_name, "-h", host, "-p", str(port), "-f", "generate_ulid_func.sql"], + stderr=STDOUT, + env={"PGPASSWORD": password}, + ).decode("utf-8") + logging.info(f"Created the ulid generation function:\n{output}") From 35084238c70bc233ba5c6e827280202471ad79e3 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 12 Nov 2024 15:55:15 +0100 Subject: [PATCH 36/42] chore: update 0.61.0 changelog --- CHANGELOG.rst | 21 +++++++++++++++++++++ helm-chart/renku/values.yaml | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1d60874a6c..9efffb52fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,28 @@ 0.61.0 ------ +Renku 0.61.0 introduces a new version of Amalthea that supports running sessions with Docker images +that do not contain Jupyter server. +NOTE to administrators: This upgrade introduces a brand new CRD for sessions. All services that support +sessions for Renku v2 will switch to this new CRD. Renku v1 sessions remain unchanged. +Therefore any old sessions for Renku v2 will not be visible to users after this upgrade. The sessions +themselves will not be immediately deleted and as long as users have saved links to their old sessions they +should be able to access their sessions and save data. However we recommend that administrators +notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and +cleaned up, rather than asking users to save the url to their sessions. + +Internal Changes +~~~~~~~~~~~~~~~~ + +**Bug Fixes** + +- **Gateway**: pass on session cookie to data services for anonymous session authentication + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-gateway 1.3.1 `_ 0.60.0 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index faefc3ea32..c28b3ae9af 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1281,7 +1281,7 @@ gateway: secretKey: image: repository: renku/renku-gateway - tag: "1.3.0" + tag: "1.3.1" pullPolicy: IfNotPresent service: type: ClusterIP From f40977ffe8cd8243528fa18bfd4c320b32318c44 Mon Sep 17 00:00:00 2001 From: Andrea Cordoba Date: Wed, 13 Nov 2024 11:45:34 +0100 Subject: [PATCH 37/42] update renku-ui to 3.42.0 --- CHANGELOG.rst | 15 ++++++++++++--- helm-chart/renku/values.yaml | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9efffb52fe..df33546b53 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,13 +7,21 @@ Renku 0.61.0 introduces a new version of Amalthea that supports running sessions that do not contain Jupyter server. NOTE to administrators: This upgrade introduces a brand new CRD for sessions. All services that support -sessions for Renku v2 will switch to this new CRD. Renku v1 sessions remain unchanged. -Therefore any old sessions for Renku v2 will not be visible to users after this upgrade. The sessions -themselves will not be immediately deleted and as long as users have saved links to their old sessions they +sessions for Renku v2 will switch to this new CRD. Renku v1 sessions remain unchanged. +Therefore any old sessions for Renku v2 will not be visible to users after this upgrade. The sessions +themselves will not be immediately deleted and as long as users have saved links to their old sessions they should be able to access their sessions and save data. However we recommend that administrators notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and cleaned up, rather than asking users to save the url to their sessions. +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**✨ Improvements** + +- **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 `__). + + Internal Changes ~~~~~~~~~~~~~~~~ @@ -25,6 +33,7 @@ Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-gateway 1.3.1 `_ +- `renku-ui 3.42.0 `_ 0.60.0 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index c28b3ae9af..62971c620a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -654,7 +654,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.41.0" + tag: "3.42.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -843,7 +843,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.41.0" + tag: "3.42.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" From c7f3a9c19b0ca2bfad66128bc62993304e752336 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 13 Nov 2024 14:19:22 +0100 Subject: [PATCH 38/42] chore: update data services to 0.26.0 --- CHANGELOG.rst | 18 +++++++++++++++--- helm-chart/renku/values.yaml | 6 +++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index df33546b53..4287c9be6d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,7 +12,9 @@ Therefore any old sessions for Renku v2 will not be visible to users after this themselves will not be immediately deleted and as long as users have saved links to their old sessions they should be able to access their sessions and save data. However we recommend that administrators notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and -cleaned up, rather than asking users to save the url to their sessions. +cleaned up, rather than asking users to save the url to their sessions. In addition to users not being able +to see old Renku v2 sessions, they will also not be able to pause, resume or delete old Renku v2 sessions. +Therefore it best is most sessions are properly saved and cleaned up before this update is rolled out. User-Facing Changes ~~~~~~~~~~~~~~~~~~~ @@ -20,20 +22,30 @@ User-Facing Changes **✨ Improvements** - **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 `__). - +- **Sessions**: Enable running session images that do not container Jupyter in them. Internal Changes ~~~~~~~~~~~~~~~~ +**Improvements** + +- **Data services**: Add support for OAuth storage providers +- **Data services**: Move notebooks code to data services + **Bug Fixes** -- **Gateway**: pass on session cookie to data services for anonymous session authentication +- **Gateway**: Pass on session cookie to data services for anonymous session authentication +- **Data services**: Correct pagination for namespaces +- **Data services**: Add creation data and created_by for namespaces +- **Data services**: Pin RClone version in data services image +- **Data services**: Properly handle multi-architecture docker images when getting working directory Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-gateway 1.3.1 `_ - `renku-ui 3.42.0 `_ +- `renku-data-services 0.26.0 `_ 0.60.0 ------ diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 62971c620a..a1d9b47c3a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -1590,14 +1590,14 @@ dataService: create: true image: repository: renku/renku-data-service - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent backgroundJobs: events: resources: {} image: repository: renku/data-service-background-jobs - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent total: resources: {} @@ -1650,7 +1650,7 @@ authz: secretsStorage: image: repository: renku/secrets-storage - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent service: type: ClusterIP From ced734168a9cc638598af823ec2c05c575b0afd1 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 13 Nov 2024 14:25:21 +0100 Subject: [PATCH 39/42] chore: update amalthea to 0.14.1 --- CHANGELOG.rst | 7 ++++++- helm-chart/renku/requirements.yaml | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4287c9be6d..6faadd0900 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,8 @@ should be able to access their sessions and save data. However we recommend that notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and cleaned up, rather than asking users to save the url to their sessions. In addition to users not being able to see old Renku v2 sessions, they will also not be able to pause, resume or delete old Renku v2 sessions. -Therefore it best is most sessions are properly saved and cleaned up before this update is rolled out. +Therefore it best is most sessions are properly saved and cleaned up before this update is rolled out. In order +to support the new CRD we have also created a new operator that will manage the new `amaltheasession` resources. User-Facing Changes ~~~~~~~~~~~~~~~~~~~ @@ -23,6 +24,7 @@ User-Facing Changes - **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 `__). - **Sessions**: Enable running session images that do not container Jupyter in them. +- **Amalthea**: Brand new operator for sessions with images that do not have to container Jupyter Internal Changes ~~~~~~~~~~~~~~~~ @@ -46,6 +48,9 @@ Individual Components - `renku-gateway 1.3.1 `_ - `renku-ui 3.42.0 `_ - `renku-data-services 0.26.0 `_ +- `amalthea 0.13.0 `_ +- `amalthea 0.14.0 `_ +- `amalthea 0.14.1 `_ 0.60.0 ------ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 01c9bb72cf..b6e2c9b59b 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,10 +23,10 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.13.0" + version: "0.14.1" - name: amalthea-sessions repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.13.0" + version: "0.14.1" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From 829feb95c1109f68823350041cf14760daefce89 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Fri, 15 Nov 2024 16:43:14 +0100 Subject: [PATCH 40/42] chore: update amalthea to 0.14.2 --- CHANGELOG.rst | 2 ++ helm-chart/renku/requirements.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6faadd0900..c022d37cc8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -41,6 +41,7 @@ Internal Changes - **Data services**: Add creation data and created_by for namespaces - **Data services**: Pin RClone version in data services image - **Data services**: Properly handle multi-architecture docker images when getting working directory +- **Amalthea**: Add readiness and health checks to sessions. Individual Components ~~~~~~~~~~~~~~~~~~~~~ @@ -51,6 +52,7 @@ Individual Components - `amalthea 0.13.0 `_ - `amalthea 0.14.0 `_ - `amalthea 0.14.1 `_ +- `amalthea 0.14.2 `_ 0.60.0 ------ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index b6e2c9b59b..f6f43daa20 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,10 +23,10 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.14.1" + version: "0.14.2" - name: amalthea-sessions repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.14.1" + version: "0.14.2" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" From be4122866915622d3ae55ff829ee59b84efe4d0e Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 18 Nov 2024 10:57:25 +0100 Subject: [PATCH 41/42] squashme: apply suggestions from code review Co-authored-by: Ralf Grubenmann --- CHANGELOG.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c022d37cc8..0a12ad2f63 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ should be able to access their sessions and save data. However we recommend that notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and cleaned up, rather than asking users to save the url to their sessions. In addition to users not being able to see old Renku v2 sessions, they will also not be able to pause, resume or delete old Renku v2 sessions. -Therefore it best is most sessions are properly saved and cleaned up before this update is rolled out. In order +Therefore it's best if most sessions are properly saved and cleaned up before this update is rolled out. In order to support the new CRD we have also created a new operator that will manage the new `amaltheasession` resources. User-Facing Changes @@ -23,8 +23,8 @@ User-Facing Changes **✨ Improvements** - **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 `__). -- **Sessions**: Enable running session images that do not container Jupyter in them. -- **Amalthea**: Brand new operator for sessions with images that do not have to container Jupyter +- **Sessions**: Enable running session images that do not contain Jupyter in them. +- **Amalthea**: Brand new operator for sessions with images that do not have to contain Jupyter Internal Changes ~~~~~~~~~~~~~~~~ @@ -38,7 +38,7 @@ Internal Changes - **Gateway**: Pass on session cookie to data services for anonymous session authentication - **Data services**: Correct pagination for namespaces -- **Data services**: Add creation data and created_by for namespaces +- **Data services**: Add creation date and created_by for namespaces - **Data services**: Pin RClone version in data services image - **Data services**: Properly handle multi-architecture docker images when getting working directory - **Amalthea**: Add readiness and health checks to sessions. From 8f9189d72fa3ece0e801c831011065b46a641ba9 Mon Sep 17 00:00:00 2001 From: olevski Date: Tue, 19 Nov 2024 08:24:02 +0100 Subject: [PATCH 42/42] chore: update amalthea to 0.14.3 --- CHANGELOG.rst | 2 ++ helm-chart/renku/requirements.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0a12ad2f63..b7f942bc3c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,6 +42,7 @@ Internal Changes - **Data services**: Pin RClone version in data services image - **Data services**: Properly handle multi-architecture docker images when getting working directory - **Amalthea**: Add readiness and health checks to sessions. +- **Amalthea**: Do not authenticate the authentication proxy health check Individual Components ~~~~~~~~~~~~~~~~~~~~~ @@ -53,6 +54,7 @@ Individual Components - `amalthea 0.14.0 `_ - `amalthea 0.14.1 `_ - `amalthea 0.14.2 `_ +- `amalthea 0.14.3 `_ 0.60.0 ------ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index f6f43daa20..f8f948b31b 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,10 +23,10 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.14.2" + version: "0.14.3" - name: amalthea-sessions repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.14.2" + version: "0.14.3" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2"