chore(postgres): Upgrade Postgres to version 16.1.0 (#3170)

Refreshes #2894 and supercedes it. Co-authored-by: Ralf Grubenmann <[email protected]>
SwissDataScienceCenter · Mar 5, 2024 · ed21d78 · ed21d78
1 parent be62726
commit ed21d78
Show file tree

Hide file tree

Showing 13 changed files with 147 additions and 87 deletions.
diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/[email protected]
-      - uses: cypress-io/github-action@v6
+      - uses: cypress-io/github-action@v5
         id: cypress
         env:
           TEST_EMAIL: [email protected]

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -5,6 +5,11 @@
 
 This release contains bug fixes to renku core service related to project migration.
 
+**Note for administrators**: this release includes breaking changes due to upgrading PostgreSQL to 16.1.0.
+This requires modifying the values file to work with the new PostgreSQL Helm chart.
+Please check (`the helm chart values changelog <https://github.com/SwissDataScienceCenter/renku/blob/master/helm-chart/values.yaml.changelog.md>`_)
+for detailed instructions.
+
 User-Facing Changes
 ~~~~~~~~~~~~~~~~~~~
 
@@ -68,6 +73,7 @@ Internal Changes
 
 **Improvements**
 
+- **Infrastructure**: Upgrade the version of PostgreSQL to 16.1.0.
 - **UI**: Add initial alpha implementation of Renku 1.0 projects
   (`#2875 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2875>`_).
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -12,9 +12,8 @@ This procedure should be followed for *any* release:
 
 * Create a release branch (e.g. `0.46.x`), if one does not already exist, with the [release action](https://github.com/SwissDataScienceCenter/renku/actions/workflows/create-release-branch.yml).
 * Create a `CHANGELOG` entry for the release and open a PR; create a deployment, this is the reference for the release.
-* Note that any PR that should go into the release needs to target the release branch _not_ `master`.
+* Note that any PR that should go into the release needs to target the release branch _not_ `master`. 
 * All release branches should be protected.
-* Use the "Rebase and Merge" button to merge release branches into `master`; do not squash commits.
 
 Acceptance tests have to pass on all release branches before merging. 
 

diff --git a/docs/how-to-guides/admin/privacycookie.rst b/docs/how-to-guides/admin/privacycookie.rst
@@ -3,11 +3,12 @@
 User interface configuration options
 ------------------------------------
 
-Privacy page and Terms of Use
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Privacy page
+~~~~~~~~~~~~
 
-The UI can be configured to show a `Privacy Policy` and `Terms of Use`. These are
-displayed under the `Help` section of the UI.
+The UI has a privacy page with a completely configurable content, suited for showing
+any policy/terms related information, like the `Privacy Policy Statement` or the
+`Terms of Use`.
 
 For each of these, the content is read from the ``privacy-and-terms`` ConfigMap.
 You need to configure the values in ``ui.client.privacy.page`` to enable the feature.
@@ -36,7 +37,7 @@ for anonymous users (i.e. without an account or not currently logged in). To com
 international laws, it's strongly advised to explicitly require consent to the user for storing
 these data and using cookies.
 
-To activate this feature, please set ``ui.privacy.banner.enabled: true``. We have already configured a
+To activate this feature, please set ``ui.privacy.enabled: true``. We have already configured a
 default cookie banner to inform the users about the aforementioned requirements and points to
 point them to the privacy page for further details.
 

diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml
@@ -1,35 +1,34 @@
 dependencies:
-- name: gitlab
-  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: 0.8.0
-  condition: gitlab.enabled
-- name: postgresql
-  version: 9.1.1
-  repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
-  condition: postgresql.enabled
-- name: keycloakx
-  version: 2.1.0
-  repository: "https://codecentric.github.io/helm-charts"
-  condition: keycloakx.enabled
-- name: redis
-  # bitnami claims that this will always contain a full set of charts - let us pray...
-  # this index was 19MB as of the date of this commit and contained redis 17.4.2
-  repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami"
-  version: 17.4.2
-  condition: redis.install
-- name: renku-jena
-  version: "0.0.23"
-  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  alias: jena
-- name: amalthea
-  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: "0.11.0"
-- name: dlf-chart
-  repository: "https://swissdatasciencecenter.github.io/datashim/"
-  version: "0.3.9-renku-2"
-  condition: notebooks.cloudstorage.s3.installDatashim
-- name: csi-rclone
-  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: "0.1.7"
-  condition: global.csi-rclone.install
-
+  - name: gitlab
+    repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+    version: 0.8.0
+    condition: gitlab.enabled
+  - name: postgresql
+    version: "14.2.4"
+    repository: "oci://registry-1.docker.io/bitnamicharts"
+    condition: postgresql.enabled
+  - name: keycloakx
+    version: 2.1.0
+    repository: "https://codecentric.github.io/helm-charts"
+    condition: keycloakx.enabled
+  - name: redis
+    # bitnami claims that this will always contain a full set of charts - let us pray...
+    # this index was 19MB as of the date of this commit and contained redis 17.4.2
+    repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami"
+    version: 17.4.2
+    condition: redis.install
+  - name: renku-jena
+    version: "0.0.23"
+    repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+    alias: jena
+  - name: amalthea
+    repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+    version: "0.11.0"
+  - name: dlf-chart
+    repository: "https://swissdatasciencecenter.github.io/datashim/"
+    version: "0.3.9-renku-2"
+    condition: notebooks.cloudstorage.s3.installDatashim
+  - name: csi-rclone
+    repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+    version: "0.1.7"
+    condition: global.csi-rclone.install
diff --git a/helm-chart/renku/templates/NOTES.txt b/helm-chart/renku/templates/NOTES.txt
@@ -9,7 +9,7 @@ can be accessed using the following one-liner (you need to have jq installed).
 kubectl get secrets -n {{ .Release.Namespace }} {{ template "renku.fullname" . }} -o json | jq -r .data.users | base64 --decode
 {{- end -}}
 
-{{ if or .Values.ui.client.privacy.banner.enabled .Values.ui.client.privacy.page.enabled -}}
+{{ if .Values.ui.client.privacy.enabled -}}
 You may need to customize privacy values for your RenkuLab deployment (E.G. the Privacy page).
 Please refer to the following documentation: https://renku.readthedocs.io/en/latest/admin/index.html#additional-configurations
 {{ end }}

diff --git a/helm-chart/renku/templates/setup-job-gitlab.yaml b/helm-chart/renku/templates/setup-job-gitlab.yaml
@@ -35,14 +35,14 @@ spec:
               {{ if .Values.global.externalServices.postgresql.enabled }}
               value: {{ .Values.global.externalServices.postgresql.username }}
               {{- else -}}
-              value: {{ .Values.postgresql.postgresqlUsername }}
+              value: {{ .Values.postgresql.auth.username }}
               {{- end }}
             {{- if not .Values.global.externalServices.postgresql.enabled }}
             - name: DB_ADMIN_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ template "postgresql.fullname" . }}"
-                  key: postgresql-password
+                  key: postgres-password
             {{- else if .Values.global.externalServices.postgresql.password }}
             - name: DB_ADMIN_PASSWORD
               value: {{ .Values.global.externalServices.postgresql.password }}
@@ -51,7 +51,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.global.externalServices.postgresql.existingSecret }}
-                  key: postgresql-password
+                  key: postgres-password
             {{- end }}
             - name: GITLAB_ENABLED
               value: {{ .Values.gitlab.enabled | quote }}

diff --git a/helm-chart/renku/templates/setup-job-keycloak-db.yaml b/helm-chart/renku/templates/setup-job-keycloak-db.yaml
@@ -35,14 +35,14 @@ spec:
               {{ if .Values.global.externalServices.postgresql.enabled }}
               value: {{ .Values.global.externalServices.postgresql.username }}
               {{- else -}}
-              value: {{ .Values.postgresql.postgresqlUsername }}
+              value: {{ .Values.postgresql.auth.username }}
               {{- end }}
             {{- if not .Values.global.externalServices.postgresql.enabled }}
             - name: DB_ADMIN_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ template "postgresql.fullname" . }}"
-                  key: postgresql-password
+                  key: postgres-password
             {{- else if .Values.global.externalServices.postgresql.password }}
             - name: DB_ADMIN_PASSWORD
               value: {{ .Values.global.externalServices.postgresql.password }}
@@ -51,7 +51,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.global.externalServices.postgresql.existingSecret }}
-                  key: postgresql-password
+                  key: postgres-password
             {{- end }}
             - name: KEYCLOAK_DB_USERNAME
               value: {{ .Values.global.keycloak.postgresUser | quote }}

diff --git a/helm-chart/renku/templates/setup-job-renku-dbs.yaml b/helm-chart/renku/templates/setup-job-renku-dbs.yaml
@@ -34,14 +34,14 @@ spec:
               {{ if .Values.global.externalServices.postgresql.enabled }}
               value: {{ .Values.global.externalServices.postgresql.username }}
               {{- else -}}
-              value: {{ .Values.postgresql.postgresqlUsername }}
+              value: {{ .Values.postgresql.auth.username }}
               {{- end }}
             {{- if not .Values.global.externalServices.postgresql.enabled }}
             - name: DB_ADMIN_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ template "postgresql.fullname" . }}"
-                  key: postgresql-password
+                  key: postgres-password
             {{- else if .Values.global.externalServices.postgresql.password }}
             - name: DB_ADMIN_PASSWORD
               value: {{ .Values.global.externalServices.postgresql.password }}
@@ -50,7 +50,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.global.externalServices.postgresql.existingSecret }}
-                  key: postgresql-password
+                  key: postgres-password
             {{- end }}
             - name: EVENTLOG_DB_USERNAME
               value: {{ .Values.global.graph.dbEventLog.postgresUser | quote }}

diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml
@@ -334,36 +334,43 @@ keycloakx:
 # For production deployments check out
 # https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values-production.yaml
 postgresql:
-  # If an external Postgres database is defined in global.externalServices.postgresql,
-  # postgresql.enabled should be false, and global.externalServices.postgresql.enabled should be true.
-  # By default, Renku-bundled Postgres is enabled.
-  enabled: true
-  ## We use the defaults here.
-  postgresqlDatabase: postgres
-  postgresqlUsername: postgres
-  ## The admin password should be set explicitly, otherwise a random string will be created.
-  ## Alternatively an existing secret can be provided. Note that postgres
-  ## DOES NOT tolerate a change of the admin password when upgrading.
-  # postgresqlPassword:
+  ## We use the defaults here. Note that these basic configs could also be set as
+  ## global values such that sub-charts can access them too.
+
+  auth:
+    username: postgres
+    database: postgres
+
+    ## The admin password should be set explicitly, otherwise a random string will be
+    ## created. Alternatively an existing secret can be provided. Note that postgres
+    ## DOES NOT tolerate a change of the admin password when upgrading.
+    # postgresqlPassword:
+
+    ## Use an existing secret instead of creating a new one. It must have a
+    ## postgresql-password key containing the password for the postgres user.
+    # existingSecret:
+
+    # Consider replication. These are the defaults for the basic settings.
+    # replicationUsername: repl_user
+    # replicationPassword: repl_password # generate a random password `openssl rand -hex 32`
+
+  # image:
+  # repository: bitnami/postgresql
+  # tag:
+
+  primary:
+    persistence:
+      ## We use the defaults here, but they will probably be modified for most deployments.
+      enabled: true
+      size: 8Gi
+      ## Provide an existing PersistentVolumeClaim to be reused.
+      # existingClaim:
+
+    # Consider replication. These are the defaults for the basic settings.
+    readReplicas:
+      enabled: false
+      replicaCount: 1
 
-  ## Use an existing secret instead of creating a new one. It must have a
-  ## postgresql-password key containing the password for the posgres user.
-  # existingSecret:
-  image:
-    repository: bitnami/postgresql
-    tag: 12.8.0
-  persistence:
-    ## We use the defaults here, but they will probably be modified for most deployments.
-    enabled: true
-    size: 8Gi
-    ## Provide an existing PersistentVolumeClaim to be reused.
-    # existingClaim:
-  # Consider replication. These are the defaults for the basic settings.
-  replication:
-    enabled: false
-    user: repl_user
-    password: repl_password # generate a random password `openssl rand -hex 32`
-    slaveReplicas: 1
 redis:
   # If set to true, a HA redis will be included in the Renku release.
   install: true
@@ -854,7 +861,8 @@ dlf-chart:
     enabled: false
   dataset-operator-chart:
     enabled: true
-csi-rclone: {}
+csi-rclone:
+  {}
   # This section is only relevant if you are installing csi-rclone as part of Renku
   ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster.
   # storageClassName: csi-rclone

diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml
@@ -27,7 +27,7 @@ spec:
           valueFrom:
             secretKeyRef:
               name: renku-postgresql ## EDIT(optional) - most likely <renku-release-name>-postgresql
-              key: postgresql-password
+              key: postgres-password
       ports:
         - containerPort: 5432
       volumeMounts:
@@ -81,7 +81,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: renku-postgresql ## EDIT(optional) - most likely <renku-release-name>-postgresql
-                  key: postgresql-password
+                  key: postgres-password
           volumeMounts:
             - mountPath: /psql-dump-data/
               name: pg-vol-tmp
diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml
@@ -28,7 +28,7 @@ spec:
           valueFrom:
             secretKeyRef:
               name: renku-postgresql ## EDIT(optional) - most likely <renku-release-name>-postgresql
-              key: postgresql-password
+              key: postgres-password
       volumeMounts:
         - mountPath: /bitnami/postgresql
           name: pg-vol-new
@@ -80,7 +80,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: renku-postgresql ## EDIT(optional) - most likely <renku-release-name>-postgresql
-                  key: postgresql-password
+                  key: postgres-password
           volumeMounts:
             - mountPath: /psql-dump-data/
               name: pg-vol-tmp