diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index fc442dbb79..40113afcf1 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -58,6 +58,7 @@ jobs: renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}} renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}} amalthea: ${{ steps.deploy-comment.outputs.amalthea}} + amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}} test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: @@ -99,6 +100,7 @@ jobs: renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}" renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}" amalthea: "${{ needs.check-deploy.outputs.amalthea }}" + amalthea_sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}" extra_values: "${{ needs.check-deploy.outputs.extra-values }}" - name: Check existing renkubot comment if: needs.check-deploy.outputs.pr-contains-string == 'true' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78b78b32d6..b7f942bc3c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,58 @@ 0.61.0 ------ +Renku 0.61.0 introduces a new version of Amalthea that supports running sessions with Docker images +that do not contain Jupyter server. + +NOTE to administrators: This upgrade introduces a brand new CRD for sessions. All services that support +sessions for Renku v2 will switch to this new CRD. Renku v1 sessions remain unchanged. +Therefore any old sessions for Renku v2 will not be visible to users after this upgrade. The sessions +themselves will not be immediately deleted and as long as users have saved links to their old sessions they +should be able to access their sessions and save data. However we recommend that administrators +notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and +cleaned up, rather than asking users to save the url to their sessions. In addition to users not being able +to see old Renku v2 sessions, they will also not be able to pause, resume or delete old Renku v2 sessions. +Therefore it's best if most sessions are properly saved and cleaned up before this update is rolled out. In order +to support the new CRD we have also created a new operator that will manage the new `amaltheasession` resources. +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**✨ Improvements** + +- **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 `__). +- **Sessions**: Enable running session images that do not contain Jupyter in them. +- **Amalthea**: Brand new operator for sessions with images that do not have to contain Jupyter + +Internal Changes +~~~~~~~~~~~~~~~~ + +**Improvements** + +- **Data services**: Add support for OAuth storage providers +- **Data services**: Move notebooks code to data services + +**Bug Fixes** + +- **Gateway**: Pass on session cookie to data services for anonymous session authentication +- **Data services**: Correct pagination for namespaces +- **Data services**: Add creation date and created_by for namespaces +- **Data services**: Pin RClone version in data services image +- **Data services**: Properly handle multi-architecture docker images when getting working directory +- **Amalthea**: Add readiness and health checks to sessions. +- **Amalthea**: Do not authenticate the authentication proxy health check + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-gateway 1.3.1 `_ +- `renku-ui 3.42.0 `_ +- `renku-data-services 0.26.0 `_ +- `amalthea 0.13.0 `_ +- `amalthea 0.14.0 `_ +- `amalthea 0.14.1 `_ +- `amalthea 0.14.2 `_ +- `amalthea 0.14.3 `_ 0.60.0 ------ @@ -79,11 +130,16 @@ Internal Changes **Bug Fixes** - **Data services**: Handle spaces in ``provider_id`` for connected services (`#482 `__). +- **csi-rclone**: Do not log potentially sensitive data in error messages. +- **csi-rclone**: Properly handle encrypted secrets with the new annotation-based storage class. + Individual Components ~~~~~~~~~~~~~~~~~~~~~ - `renku-data-services 0.24.2 `__ +- `csi-rclone 0.3.4 `__ +- `csi-rclone 0.3.5 `__ 0.59.1 ------ diff --git a/cypress-tests/cypress/e2e/useSession.cy.ts b/cypress-tests/cypress/e2e/useSession.cy.ts index 30d7662e7d..537e7f6bf2 100644 --- a/cypress-tests/cypress/e2e/useSession.cy.ts +++ b/cypress-tests/cypress/e2e/useSession.cy.ts @@ -189,6 +189,8 @@ describe("Basic public project functionality", () => { // Stop the session -- mind that anonymous users cannot pause sessions cy.deleteSession({ fromSessionPage: true }); + + cy.robustLogin(); }); it("Start a new session on a project without permissions.", () => { diff --git a/cypress-tests/cypress/support/utils/projectsV2.utils.ts b/cypress-tests/cypress/support/utils/projectsV2.utils.ts index 1770aaff91..e55665f105 100644 --- a/cypress-tests/cypress/support/utils/projectsV2.utils.ts +++ b/cypress-tests/cypress/support/utils/projectsV2.utils.ts @@ -23,7 +23,7 @@ export function getUserNamespaceAPIV2(): Cypress.Chainable { /** Get a project by using only the API. */ export function getProjectByNamespaceAPIV2(newProjectProps: ProjectIdentifierV2): Cypress.Chainable { - return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/projects/${newProjectProps.namespace}/${newProjectProps.slug}` }); + return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/namespaces/${newProjectProps.namespace}/projects/${newProjectProps.slug}` }); } /** Create a project (if the project is missing) by using only the API. */ diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index c1c773a880..f8f948b31b 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,7 +23,10 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.12.3" + version: "0.14.3" + - name: amalthea-sessions + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.14.3" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" diff --git a/helm-chart/renku/templates/_helpers.tpl b/helm-chart/renku/templates/_helpers.tpl index c7d07db31d..f7660399d1 100644 --- a/helm-chart/renku/templates/_helpers.tpl +++ b/helm-chart/renku/templates/_helpers.tpl @@ -149,6 +149,10 @@ KC_DB_PASSWORD: {{ default (randAlphaNum 64) .Values.global.keycloak.postgresPas {{- end -}} {{- end -}} +{{- define "renku.keycloakIssuerUrl" -}} +{{- printf "%s/realms/%s" (include "renku.keycloakUrl" . | trimSuffix "/") (include "renku.keycloak.realm" .) -}} +{{- end -}} + {{/* Common labels */}} diff --git a/helm-chart/renku/templates/data-service/deployment.yaml b/helm-chart/renku/templates/data-service/deployment.yaml index a0620c6f78..e657cd9b4f 100644 --- a/helm-chart/renku/templates/data-service/deployment.yaml +++ b/helm-chart/renku/templates/data-service/deployment.yaml @@ -43,6 +43,9 @@ spec: - name: http containerPort: 8000 protocol: TCP + envFrom: + - secretRef: + name: {{ template "renku.notebooks.fullname" . }} env: - name: VERSION value: {{ .Values.dataService.image.tag | quote }} @@ -70,10 +73,6 @@ spec: value: /secrets/encryptionKey/encryptionKey - name: SECRETS_SERVICE_PUBLIC_KEY_PATH value: /secrets/publicKey/publicKey - - name: SERVER_DEFAULTS - value: /etc/renku-data-service/server_options/server_defaults.json - - name: SERVER_OPTIONS - value: /etc/renku-data-service/server_options/server_options.json - name: K8S_NAMESPACE value: {{ .Release.Namespace | quote }} - name: GITLAB_URL @@ -116,11 +115,21 @@ spec: value: {{ .Values.dataService.trustedProxies.proxiesCount | default "" | quote }} - name: REAL_IP_HEADER value: {{ .Values.dataService.trustedProxies.realIpHeader | default "" | quote }} + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: SESSIONS_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace {{- include "certificates.env.python" . | nindent 12 }} {{- include "certificates.env.grpc" . | nindent 12 }} volumeMounts: - name: server-options - mountPath: /etc/renku-data-service/server_options + mountPath: /etc/renku-notebooks/server_options - mountPath: "/secrets/encryptionKey" name: encryption-key readOnly: true diff --git a/helm-chart/renku/templates/data-service/rbac.yaml b/helm-chart/renku/templates/data-service/rbac.yaml index c242893a24..2e8266a050 100644 --- a/helm-chart/renku/templates/data-service/rbac.yaml +++ b/helm-chart/renku/templates/data-service/rbac.yaml @@ -19,6 +19,67 @@ rules: - patch - delete - create + - apiGroups: + - "" + resources: + - pods + - pods/log + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - delete + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch + - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - delete + - patch + - apiGroups: + - {{ .Values.amalthea.crdApiGroup }} + resources: + - {{ .Values.amalthea.crdNames.plural }} + verbs: + - create + - update + - delete + - patch + - list + - get + - watch + - apiGroups: + - amalthea.dev + resources: + - amaltheasessions + verbs: + - create + - update + - delete + - patch + - list + - get + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/helm-chart/renku/templates/notebooks/deployment.yaml b/helm-chart/renku/templates/notebooks/deployment.yaml index 8e65a6d313..978d466f12 100644 --- a/helm-chart/renku/templates/notebooks/deployment.yaml +++ b/helm-chart/renku/templates/notebooks/deployment.yaml @@ -43,6 +43,12 @@ spec: value: {{ .Values.amalthea.crdApiVersion }} - name: K8S_WATCHER_CR_PLURAL value: {{ .Values.amalthea.crdNames.plural }} + - name: K8S_WATCHER_AMALTHEA_SESSION_GROUP + value: amalthea.dev + - name: K8S_WATCHER_AMALTHEA_SESSION_VERSION + value: v1alpha1 + - name: K8S_WATCHER_AMALTHEA_SESSION_PLURAL + value: amaltheasessions - name: K8S_WATCHER_NAMESPACES {{ if .Values.notebooks.sessionsNamespace }} value: {{ list .Release.Namespace .Values.notebooks.sessionsNamespace | uniq | toJson | quote }} diff --git a/helm-chart/renku/templates/notebooks/env-secret.yaml b/helm-chart/renku/templates/notebooks/env-secret.yaml new file mode 100644 index 0000000000..c229aab657 --- /dev/null +++ b/helm-chart/renku/templates/notebooks/env-secret.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "renku.notebooks.fullname" . }} + labels: + chart: {{ template "renku.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +stringData: + NB_SESSIONS__STORAGE__PVS_ENABLED: {{ .Values.notebooks.userSessionPersistentVolumes.enabled | quote }} + {{ if .Values.notebooks.userSessionPersistentVolumes.enabled }} + NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS: {{ .Values.notebooks.userSessionPersistentVolumes.storageClass | quote}} + {{ end }} + NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT: {{ .Values.notebooks.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }} + NB_SESSIONS__DEFAULT_IMAGE: "{{ .Values.notebooks.defaultSessionImage }}" + NB_SERVER_OPTIONS__DEFAULTS_PATH: /etc/renku-notebooks/server_options/server_defaults.json + NB_SERVER_OPTIONS__UI_CHOICES_PATH: /etc/renku-notebooks/server_options/server_options.json + NB_SESSIONS__OIDC__CLIENT_ID: {{ .Values.notebooks.oidc.clientId }} + NB_SESSIONS__OIDC__CLIENT_SECRET: {{ .Values.notebooks.oidc.clientSecret }} + NB_SESSIONS__OIDC__AUTH_URL: {{ .Values.notebooks.oidc.authUrl }} + NB_SESSIONS__OIDC__TOKEN_URL: {{ .Values.notebooks.oidc.tokenUrl }} + NB_SESSIONS__OIDC__ISSUER_URL: {{ template "renku.keycloakIssuerUrl" . }} + NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }} + NB_SESSIONS__INGRESS__HOST: {{ .Values.notebooks.sessionIngress.host }} + NB_SESSIONS__INGRESS__TLS_SECRET: {{ .Values.notebooks.sessionIngress.tlsSecret }} + NB_SESSIONS__INGRESS__ANNOTATIONS: | + {{- .Values.notebooks.sessionIngress.annotations | toYaml | nindent 4 }} + NB_GIT__URL: {{ .Values.global.gitlab.url | quote }} + NB_GIT__REGISTRY: {{ required "An image registry must be specified." .Values.global.gitlab.registry.host }} + NB_SESSIONS__GIT_RPC_SERVER__IMAGE: "{{ .Values.notebooks.gitRpcServer.image.name }}:{{ .Values.notebooks.gitRpcServer.image.tag }}" + NB_SESSIONS__GIT_PROXY__IMAGE: "{{ .Values.notebooks.gitHttpsProxy.image.name }}:{{ .Values.notebooks.gitHttpsProxy.image.tag }}" + NB_SESSIONS__GIT_CLONE__IMAGE: "{{ .Values.notebooks.gitClone.image.name }}:{{ .Values.notebooks.gitClone.image.tag }}" + NB_ANONYMOUS_SESSIONS_ENABLED: {{ .Values.global.anonymousSessions.enabled | quote }} + NB_SSH_ENABLED: {{ .Values.notebooks.ssh.enabled | quote }} + NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.anonymous | quote }} + NB_SESSIONS__CULLING__REGISTERED__HIBERNATED_SECONDS: {{ .Values.notebooks.culling.hibernatedSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.registered | quote }} + NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.anonymous | quote }} + NB_AMALTHEA__GROUP: {{ .Values.amalthea.crdApiGroup }} + NB_AMALTHEA__VERSION: {{ .Values.amalthea.crdApiVersion }} + NB_AMALTHEA__PLURAL: {{ .Values.amalthea.crdNames.plural }} + NB_AMALTHEA__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher" + NB_AMALTHEA_V2__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher" + NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }} + NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.gitClone.sampleRate | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.sidecar.enabled | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.sidecar.dsn | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.sidecar.environment | quote }} + NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.sidecar.sampleRate | quote }} + NB_SESSIONS__CA_CERTS__IMAGE: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}" + NB_SESSIONS__CA_CERTS__SECRETS: | + {{- .Values.global.certificates.customCAs | toYaml | nindent 4 }} + {{- with .Values.notebooks.sessionNodeSelector }} + NB_SESSIONS__NODE_SELECTOR: | + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.notebooks.sessionAffinity }} + NB_SESSIONS__AFFINITY: | + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.notebooks.sessionTolerations }} + NB_SESSIONS__TOLERATIONS: | + {{- toYaml . | nindent 4 }} + {{- end }} + NB_SESSIONS__ENFORCE_CPU_LIMITS: {{ .Values.notebooks.enforceCPULimits | quote }} + NB_CLOUD_STORAGE__ENABLED: {{ .Values.notebooks.cloudstorage.enabled | quote }} + NB_CLOUD_STORAGE__STORAGE_CLASS: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} + NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} + NB_VERSION: {{ .Values.notebooks.image.tag | quote }} + {{ if .Values.notebooks.sessionsNamespace }} + NB_K8S__SESSIONS_NAMESPACE: {{ .Values.notebooks.sessionsNamespace | quote }} + {{ end }} + NB_K8S__RENKU_NAMESPACE: {{ .Release.Namespace | quote }} + NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_ID: renku + NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_SECRET: {{ .Values.global.gateway.clientSecret | quote }} + NB_KEYCLOAK_REALM: {{ include "renku.keycloak.realm" . | quote }} + NB_SESSIONS__SSH__ENABLED: {{ .Values.notebooks.ssh.enabled | quote }} + {{- if not (kindIs "invalid" .Values.notebooks.ssh.hostKeySecret) }} + NB_SESSIONS__SSH__HOST_KEY_SECRET: {{ .Values.notebooks.ssh.hostKeySecret | quote }} + {{- end }} + NB_DATA_SERVICE_URL: {{ printf "http://%s-data-service/api/data" .Release.Name }} + NB_USER_SECRETS__SECRETS_STORAGE_SERVICE_URL: {{ printf "http://%s-secrets-storage" .Release.Name }} + NB_USER_SECRETS__IMAGE: "{{ .Values.notebooks.secretsMount.image.repository}}:{{.Values.notebooks.secretsMount.image.tag }}" +--- + diff --git a/helm-chart/renku/templates/notebooks/network-policy.yaml b/helm-chart/renku/templates/notebooks/network-policy.yaml index 7de4810309..7f954dad17 100644 --- a/helm-chart/renku/templates/notebooks/network-policy.yaml +++ b/helm-chart/renku/templates/notebooks/network-policy.yaml @@ -15,6 +15,10 @@ spec: matchLabels: app: {{ template "renku.notebooks.name" . }} release: {{ .Release.Name }} + - podSelector: + matchLabels: + app: renku-data-service + release: {{ .Release.Name }} ports: - protocol: TCP port: http diff --git a/helm-chart/renku/templates/notebooks/statefulset.yaml b/helm-chart/renku/templates/notebooks/statefulset.yaml index a8de45596e..04a887f900 100644 --- a/helm-chart/renku/templates/notebooks/statefulset.yaml +++ b/helm-chart/renku/templates/notebooks/statefulset.yaml @@ -32,74 +32,14 @@ spec: imagePullPolicy: {{ .Values.notebooks.image.pullPolicy }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + envFrom: + - secretRef: + name: {{ template "renku.notebooks.fullname" . }} env: - - name: NB_SESSIONS__STORAGE__PVS_ENABLED - value: {{ .Values.notebooks.userSessionPersistentVolumes.enabled | quote }} - {{ if .Values.notebooks.userSessionPersistentVolumes.enabled }} - - name: NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS - value: {{ .Values.notebooks.userSessionPersistentVolumes.storageClass | quote}} - {{ end }} - - name: NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT - value: {{ .Values.notebooks.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }} - - name: NB_SESSIONS__DEFAULT_IMAGE - value: "{{ .Values.notebooks.defaultSessionImage }}" - - name: NB_SERVER_OPTIONS__DEFAULTS_PATH - value: /etc/renku-notebooks/server_options/server_defaults.json - - name: NB_SERVER_OPTIONS__UI_CHOICES_PATH - value: /etc/renku-notebooks/server_options/server_options.json {{ if eq .Values.global.debug true }} - name: FLASK_DEBUG value: "1" {{ end }} - - name: NB_SESSIONS__OIDC__CLIENT_ID - value: {{ .Values.notebooks.oidc.clientId }} - - name: NB_SESSIONS__OIDC__CLIENT_SECRET - value: {{ .Values.notebooks.oidc.clientSecret }} - - name: NB_SESSIONS__OIDC__AUTH_URL - value: {{ .Values.notebooks.oidc.authUrl }} - - name: NB_SESSIONS__OIDC__TOKEN_URL - value: {{ .Values.notebooks.oidc.tokenUrl }} - - name: NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL - value: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }} - - name: NB_SESSIONS__INGRESS__HOST - value: {{ .Values.notebooks.sessionIngress.host }} - - name: NB_SESSIONS__INGRESS__TLS_SECRET - value: {{ .Values.notebooks.sessionIngress.tlsSecret }} - - name: NB_SESSIONS__INGRESS__ANNOTATIONS - value: | - {{- .Values.notebooks.sessionIngress.annotations | toYaml | nindent 16 }} - - name: NB_GIT__URL - value: {{ .Values.global.gitlab.url | quote }} - - name: NB_GIT__REGISTRY - value: {{ required "An image registry must be specified." .Values.global.gitlab.registry.host }} - - name: NB_SESSIONS__GIT_RPC_SERVER__IMAGE - value: "{{ .Values.notebooks.gitRpcServer.image.name }}:{{ .Values.notebooks.gitRpcServer.image.tag }}" - - name: NB_SESSIONS__GIT_PROXY__IMAGE - value: "{{ .Values.notebooks.gitHttpsProxy.image.name }}:{{ .Values.notebooks.gitHttpsProxy.image.tag }}" - - name: NB_SESSIONS__GIT_CLONE__IMAGE - value: "{{ .Values.notebooks.gitClone.image.name }}:{{ .Values.notebooks.gitClone.image.tag }}" - - name: NB_ANONYMOUS_SESSIONS_ENABLED - value: {{ .Values.global.anonymousSessions.enabled | quote }} - - name: NB_SSH_ENABLED - value: {{ .Values.notebooks.ssh.enabled | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS - value: {{ .Values.notebooks.culling.idleSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS - value: {{ .Values.notebooks.culling.idleSecondsThreshold.anonymous | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__HIBERNATED_SECONDS - value: {{ .Values.notebooks.culling.hibernatedSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS - value: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.registered | quote }} - - name: NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS - value: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.anonymous | quote }} - - name: NB_AMALTHEA__GROUP - value: {{ .Values.amalthea.crdApiGroup }} - - name: NB_AMALTHEA__VERSION - value: {{ .Values.amalthea.crdApiVersion }} - - name: NB_AMALTHEA__PLURAL - value: {{ .Values.amalthea.crdNames.plural }} - - name: NB_AMALTHEA__CACHE_URL - value: http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher - name: NB_SENTRY__ENABLED value: {{ .Values.notebooks.sentry.enabled | quote }} - name: NB_SENTRY__DSN @@ -110,86 +50,14 @@ spec: value: {{ .Values.notebooks.sentry.sampleRate | quote }} - name: SENTRY_RELEASE value: {{ .Chart.Version | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED - value: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__DSN - value: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__ENV - value: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }} - - name: NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE - value: {{ .Values.notebooks.sessionSentry.gitClone.sampleRate | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED - value: {{ .Values.notebooks.sessionSentry.sidecar.enabled | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN - value: {{ .Values.notebooks.sessionSentry.sidecar.dsn | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV - value: {{ .Values.notebooks.sessionSentry.sidecar.environment | quote }} - - name: NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE - value: {{ .Values.notebooks.sessionSentry.sidecar.sampleRate | quote }} - - name: NB_SESSIONS__CA_CERTS__IMAGE - value: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}" - - name: NB_SESSIONS__CA_CERTS__SECRETS - value: | - {{- .Values.global.certificates.customCAs | toYaml | nindent 16 }} - {{- with .Values.notebooks.sessionNodeSelector }} - - name: NB_SESSIONS__NODE_SELECTOR - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - {{- with .Values.notebooks.sessionAffinity }} - - name: NB_SESSIONS__AFFINITY - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - {{- with .Values.notebooks.sessionTolerations }} - - name: NB_SESSIONS__TOLERATIONS - value: | - {{- toYaml . | nindent 16 }} - {{- end }} - name: KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace {{- include "certificates.env.python" . | nindent 12 }} - - name: NB_SESSIONS__ENFORCE_CPU_LIMITS - value: {{ .Values.notebooks.enforceCPULimits | quote }} - - name: NB_CLOUD_STORAGE__ENABLED - value: {{ .Values.notebooks.cloudstorage.enabled | quote }} - - name: NB_CLOUD_STORAGE__STORAGE_CLASS - value: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} - - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS - value: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} - - name: NB_VERSION - value: {{ .Values.notebooks.image.tag | quote }} - {{ if .Values.notebooks.sessionsNamespace }} - - name: NB_K8S__SESSIONS_NAMESPACE - value: {{ .Values.notebooks.sessionsNamespace | quote }} - {{ end }} - - name: NB_K8S__BYPASS_CACHE_ON_FAILURE - value: {{ .Values.notebooks.bypassCacheOnFailure | quote }} - - name: NB_K8S__RENKU_NAMESPACE - value: {{ .Release.Namespace | quote }} - - name: NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_ID - value: renku - - name: NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_SECRET - value: {{ .Values.global.gateway.clientSecret | quote }} - - name: NB_KEYCLOAK_REALM - value: {{ include "renku.keycloak.realm" . | quote }} - - name: NB_SESSIONS__SSH__ENABLED - value: {{ .Values.notebooks.ssh.enabled | quote }} - {{- if not (kindIs "invalid" .Values.notebooks.ssh.hostKeySecret) }} - - name: NB_SESSIONS__SSH__HOST_KEY_SECRET - value: {{ .Values.notebooks.ssh.hostKeySecret | quote }} - {{- end }} - name: NB_DUMMY_STORES value: {{ .Values.notebooks.dummyStores | quote }} - - name: NB_DATA_SERVICE_URL - value: {{ printf "http://%s-data-service/api/data" .Release.Name}} - - name: NB_USER_SECRETS__SECRETS_STORAGE_SERVICE_URL - value: {{ printf "http://%s-secrets-storage" .Release.Name}} - - name: NB_USER_SECRETS__IMAGE - value: "{{ .Values.notebooks.secretsMount.image.repository}}:{{.Values.notebooks.secretsMount.image.tag}}" ports: - name: http containerPort: 8000 diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index faefc3ea32..a1d9b47c3a 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -654,7 +654,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.41.0" + tag: "3.42.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -843,7 +843,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.41.0" + tag: "3.42.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" @@ -1281,7 +1281,7 @@ gateway: secretKey: image: repository: renku/renku-gateway - tag: "1.3.0" + tag: "1.3.1" pullPolicy: IfNotPresent service: type: ClusterIP @@ -1590,14 +1590,14 @@ dataService: create: true image: repository: renku/renku-data-service - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent backgroundJobs: events: resources: {} image: repository: renku/data-service-background-jobs - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent total: resources: {} @@ -1650,7 +1650,7 @@ authz: secretsStorage: image: repository: renku/secrets-storage - tag: "0.25.0" + tag: "0.26.0" pullPolicy: IfNotPresent service: type: ClusterIP diff --git a/scripts/init-db/Dockerfile b/scripts/init-db/Dockerfile index fb2b22deb4..e194593dd5 100644 --- a/scripts/init-db/Dockerfile +++ b/scripts/init-db/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.11-slim-bullseye RUN apt-get update && apt-get install -y \ + postgresql-client \ tini && \ rm -rf /var/lib/apt/lists/* COPY . . diff --git a/scripts/init-db/generate_ulid_func.sql b/scripts/init-db/generate_ulid_func.sql new file mode 100644 index 0000000000..90b2aa8305 --- /dev/null +++ b/scripts/init-db/generate_ulid_func.sql @@ -0,0 +1,83 @@ +-- From https://github.com/geckoboard/pgulid/blob/master/pgulid.sql +-- Taken at commit sha b265253 +-- pgulid is based on OK Log's Go implementation of the ULID spec +-- +-- https://github.com/oklog/ulid +-- https://github.com/ulid/spec +-- +-- Copyright 2016 The Oklog Authors +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- NOTE: REPLACE will error if you change the name, args or return type of the function +-- There is no CREATE IF EXISTS, this is the closest thing that gives similar functionality +CREATE OR REPLACE FUNCTION generate_ulid() +RETURNS TEXT +AS $$ +DECLARE + -- Crockford's Base32 + encoding BYTEA = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + timestamp BYTEA = E'\\000\\000\\000\\000\\000\\000'; + output TEXT = ''; + + unix_time BIGINT; + ulid BYTEA; +BEGIN + -- 6 timestamp bytes + unix_time = (EXTRACT(EPOCH FROM CLOCK_TIMESTAMP()) * 1000)::BIGINT; + timestamp = SET_BYTE(timestamp, 0, (unix_time >> 40)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 1, (unix_time >> 32)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 2, (unix_time >> 24)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 3, (unix_time >> 16)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 4, (unix_time >> 8)::BIT(8)::INTEGER); + timestamp = SET_BYTE(timestamp, 5, unix_time::BIT(8)::INTEGER); + + -- 10 entropy bytes + ulid = timestamp || gen_random_bytes(10); + + -- Encode the timestamp + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 224) >> 5)); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 31))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 1) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 1) & 7) << 2) | ((GET_BYTE(ulid, 2) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 2) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 2) & 1) << 4) | ((GET_BYTE(ulid, 3) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 3) & 15) << 1) | ((GET_BYTE(ulid, 4) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 4) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 4) & 3) << 3) | ((GET_BYTE(ulid, 5) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 5) & 31))); + + -- Encode the entropy + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 6) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 6) & 7) << 2) | ((GET_BYTE(ulid, 7) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 7) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 7) & 1) << 4) | ((GET_BYTE(ulid, 8) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 8) & 15) << 1) | ((GET_BYTE(ulid, 9) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 9) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 9) & 3) << 3) | ((GET_BYTE(ulid, 10) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 10) & 31))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 11) & 248) >> 3)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 11) & 7) << 2) | ((GET_BYTE(ulid, 12) & 192) >> 6))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 12) & 62) >> 1)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 12) & 1) << 4) | ((GET_BYTE(ulid, 13) & 240) >> 4))); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 13) & 15) << 1) | ((GET_BYTE(ulid, 14) & 128) >> 7))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 14) & 124) >> 2)); + output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 14) & 3) << 3) | ((GET_BYTE(ulid, 15) & 224) >> 5))); + output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 15) & 31))); + + RETURN output; +END +$$ +LANGUAGE plpgsql +VOLATILE; diff --git a/scripts/init-db/renku_db_init.py b/scripts/init-db/renku_db_init.py index e4789b38e0..0e2bfc31c0 100644 --- a/scripts/init-db/renku_db_init.py +++ b/scripts/init-db/renku_db_init.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from queries import DatabaseInit -from utils import get_db_connection +from utils import create_ulid_func, get_db_connection logging.basicConfig(level=logging.INFO) @@ -132,7 +132,7 @@ def main(): config.renku_db_name, config.renku_db_password, postgres_db_connection, - ["pg_trgm"], + ["pg_trgm", "pgcrypto"], config.db_admin_username, ) db_init.create_database() @@ -147,6 +147,7 @@ def main(): renku_conn.set_session(autocommit=True) db_init.set_connection(renku_conn) db_init.set_extensions_and_roles() + create_ulid_func(config.db_admin_username, config.db_admin_password, config.renku_db_name, config.db_host, config.db_port) if __name__ == "__main__": diff --git a/scripts/init-db/utils.py b/scripts/init-db/utils.py index 10355b8877..ffb09e8e5f 100644 --- a/scripts/init-db/utils.py +++ b/scripts/init-db/utils.py @@ -1,4 +1,5 @@ import logging +from subprocess import STDOUT, check_output import requests @@ -36,3 +37,12 @@ def gitlab_is_online(url: str) -> int: f"Gitlab is not available at {url}, status code is {res.status_code}" ) return res.status_code + + +def create_ulid_func(username: str, password: str, db_name: str, host: str, port: int): + output = check_output( + ["psql", "-U", username, "-d", db_name, "-h", host, "-p", str(port), "-f", "generate_ulid_func.sql"], + stderr=STDOUT, + env={"PGPASSWORD": password}, + ).decode("utf-8") + logging.info(f"Created the ulid generation function:\n{output}")