From ed21d7826de5463671243418860195c4957c0606 Mon Sep 17 00:00:00 2001 From: Alessandro Degano <40891147+aledegano@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:33:34 +0100 Subject: [PATCH] chore(postgres): Upgrade Postgres to version 16.1.0 (#3170) Refreshes #2894 and supercedes it. Co-authored-by: Ralf Grubenmann --- .github/workflows/renku-dev-test.yaml | 2 +- CHANGELOG.rst | 6 ++ RELEASE.md | 3 +- docs/how-to-guides/admin/privacycookie.rst | 11 +-- helm-chart/renku/requirements.yaml | 67 +++++++++--------- helm-chart/renku/templates/NOTES.txt | 2 +- .../renku/templates/setup-job-gitlab.yaml | 6 +- .../templates/setup-job-keycloak-db.yaml | 6 +- .../renku/templates/setup-job-renku-dbs.yaml | 6 +- helm-chart/renku/values.yaml | 68 +++++++++++-------- .../version_upgrades/psql_dump.yaml | 4 +- .../version_upgrades/psql_load.yaml | 4 +- helm-chart/values.yaml.changelog.md | 49 ++++++++++++- 13 files changed, 147 insertions(+), 87 deletions(-) diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index d02c960016..9f922efd99 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4.1.1 - - uses: cypress-io/github-action@v6 + - uses: cypress-io/github-action@v5 id: cypress env: TEST_EMAIL: renku@datascience.ch diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 410cac348b..19046d1e32 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,11 @@ This release contains bug fixes to renku core service related to project migration. +**Note for administrators**: this release includes breaking changes due to upgrading PostgreSQL to 16.1.0. +This requires modifying the values file to work with the new PostgreSQL Helm chart. +Please check (`the helm chart values changelog `_) +for detailed instructions. + User-Facing Changes ~~~~~~~~~~~~~~~~~~~ @@ -68,6 +73,7 @@ Internal Changes **Improvements** +- **Infrastructure**: Upgrade the version of PostgreSQL to 16.1.0. - **UI**: Add initial alpha implementation of Renku 1.0 projects (`#2875 `_). diff --git a/RELEASE.md b/RELEASE.md index 08b6caeee1..e552b2ef0f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,9 +12,8 @@ This procedure should be followed for *any* release: * Create a release branch (e.g. `0.46.x`), if one does not already exist, with the [release action](https://github.com/SwissDataScienceCenter/renku/actions/workflows/create-release-branch.yml). * Create a `CHANGELOG` entry for the release and open a PR; create a deployment, this is the reference for the release. -* Note that any PR that should go into the release needs to target the release branch _not_ `master`. +* Note that any PR that should go into the release needs to target the release branch _not_ `master`. * All release branches should be protected. -* Use the "Rebase and Merge" button to merge release branches into `master`; do not squash commits. Acceptance tests have to pass on all release branches before merging. diff --git a/docs/how-to-guides/admin/privacycookie.rst b/docs/how-to-guides/admin/privacycookie.rst index 1e2472f7ed..d76f4a0411 100644 --- a/docs/how-to-guides/admin/privacycookie.rst +++ b/docs/how-to-guides/admin/privacycookie.rst @@ -3,11 +3,12 @@ User interface configuration options ------------------------------------ -Privacy page and Terms of Use -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Privacy page +~~~~~~~~~~~~ -The UI can be configured to show a `Privacy Policy` and `Terms of Use`. These are -displayed under the `Help` section of the UI. +The UI has a privacy page with a completely configurable content, suited for showing +any policy/terms related information, like the `Privacy Policy Statement` or the +`Terms of Use`. For each of these, the content is read from the ``privacy-and-terms`` ConfigMap. You need to configure the values in ``ui.client.privacy.page`` to enable the feature. @@ -36,7 +37,7 @@ for anonymous users (i.e. without an account or not currently logged in). To com international laws, it's strongly advised to explicitly require consent to the user for storing these data and using cookies. -To activate this feature, please set ``ui.privacy.banner.enabled: true``. We have already configured a +To activate this feature, please set ``ui.privacy.enabled: true``. We have already configured a default cookie banner to inform the users about the aforementioned requirements and points to point them to the privacy page for further details. diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 7ce40f7f13..b9b52d57cc 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -1,35 +1,34 @@ dependencies: -- name: gitlab - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: 0.8.0 - condition: gitlab.enabled -- name: postgresql - version: 9.1.1 - repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami" - condition: postgresql.enabled -- name: keycloakx - version: 2.1.0 - repository: "https://codecentric.github.io/helm-charts" - condition: keycloakx.enabled -- name: redis - # bitnami claims that this will always contain a full set of charts - let us pray... - # this index was 19MB as of the date of this commit and contained redis 17.4.2 - repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" - version: 17.4.2 - condition: redis.install -- name: renku-jena - version: "0.0.23" - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - alias: jena -- name: amalthea - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.11.0" -- name: dlf-chart - repository: "https://swissdatasciencecenter.github.io/datashim/" - version: "0.3.9-renku-2" - condition: notebooks.cloudstorage.s3.installDatashim -- name: csi-rclone - repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.1.7" - condition: global.csi-rclone.install - + - name: gitlab + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: 0.8.0 + condition: gitlab.enabled + - name: postgresql + version: "14.2.4" + repository: "oci://registry-1.docker.io/bitnamicharts" + condition: postgresql.enabled + - name: keycloakx + version: 2.1.0 + repository: "https://codecentric.github.io/helm-charts" + condition: keycloakx.enabled + - name: redis + # bitnami claims that this will always contain a full set of charts - let us pray... + # this index was 19MB as of the date of this commit and contained redis 17.4.2 + repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" + version: 17.4.2 + condition: redis.install + - name: renku-jena + version: "0.0.23" + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + alias: jena + - name: amalthea + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.11.0" + - name: dlf-chart + repository: "https://swissdatasciencecenter.github.io/datashim/" + version: "0.3.9-renku-2" + condition: notebooks.cloudstorage.s3.installDatashim + - name: csi-rclone + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.1.7" + condition: global.csi-rclone.install diff --git a/helm-chart/renku/templates/NOTES.txt b/helm-chart/renku/templates/NOTES.txt index d0ab572768..c1ea9777d7 100644 --- a/helm-chart/renku/templates/NOTES.txt +++ b/helm-chart/renku/templates/NOTES.txt @@ -9,7 +9,7 @@ can be accessed using the following one-liner (you need to have jq installed). kubectl get secrets -n {{ .Release.Namespace }} {{ template "renku.fullname" . }} -o json | jq -r .data.users | base64 --decode {{- end -}} -{{ if or .Values.ui.client.privacy.banner.enabled .Values.ui.client.privacy.page.enabled -}} +{{ if .Values.ui.client.privacy.enabled -}} You may need to customize privacy values for your RenkuLab deployment (E.G. the Privacy page). Please refer to the following documentation: https://renku.readthedocs.io/en/latest/admin/index.html#additional-configurations {{ end }} diff --git a/helm-chart/renku/templates/setup-job-gitlab.yaml b/helm-chart/renku/templates/setup-job-gitlab.yaml index f01a4747d5..b6fc4cc58c 100644 --- a/helm-chart/renku/templates/setup-job-gitlab.yaml +++ b/helm-chart/renku/templates/setup-job-gitlab.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: GITLAB_ENABLED value: {{ .Values.gitlab.enabled | quote }} diff --git a/helm-chart/renku/templates/setup-job-keycloak-db.yaml b/helm-chart/renku/templates/setup-job-keycloak-db.yaml index 4d3229b2f0..68736d5441 100644 --- a/helm-chart/renku/templates/setup-job-keycloak-db.yaml +++ b/helm-chart/renku/templates/setup-job-keycloak-db.yaml @@ -35,14 +35,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -51,7 +51,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: KEYCLOAK_DB_USERNAME value: {{ .Values.global.keycloak.postgresUser | quote }} diff --git a/helm-chart/renku/templates/setup-job-renku-dbs.yaml b/helm-chart/renku/templates/setup-job-renku-dbs.yaml index 50a49b7a7a..45dda94363 100644 --- a/helm-chart/renku/templates/setup-job-renku-dbs.yaml +++ b/helm-chart/renku/templates/setup-job-renku-dbs.yaml @@ -34,14 +34,14 @@ spec: {{ if .Values.global.externalServices.postgresql.enabled }} value: {{ .Values.global.externalServices.postgresql.username }} {{- else -}} - value: {{ .Values.postgresql.postgresqlUsername }} + value: {{ .Values.postgresql.auth.username }} {{- end }} {{- if not .Values.global.externalServices.postgresql.enabled }} - name: DB_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{{ template "postgresql.fullname" . }}" - key: postgresql-password + key: postgres-password {{- else if .Values.global.externalServices.postgresql.password }} - name: DB_ADMIN_PASSWORD value: {{ .Values.global.externalServices.postgresql.password }} @@ -50,7 +50,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.global.externalServices.postgresql.existingSecret }} - key: postgresql-password + key: postgres-password {{- end }} - name: EVENTLOG_DB_USERNAME value: {{ .Values.global.graph.dbEventLog.postgresUser | quote }} diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 2062843433..113760eee1 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -334,36 +334,43 @@ keycloakx: # For production deployments check out # https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values-production.yaml postgresql: - # If an external Postgres database is defined in global.externalServices.postgresql, - # postgresql.enabled should be false, and global.externalServices.postgresql.enabled should be true. - # By default, Renku-bundled Postgres is enabled. - enabled: true - ## We use the defaults here. - postgresqlDatabase: postgres - postgresqlUsername: postgres - ## The admin password should be set explicitly, otherwise a random string will be created. - ## Alternatively an existing secret can be provided. Note that postgres - ## DOES NOT tolerate a change of the admin password when upgrading. - # postgresqlPassword: + ## We use the defaults here. Note that these basic configs could also be set as + ## global values such that sub-charts can access them too. + + auth: + username: postgres + database: postgres + + ## The admin password should be set explicitly, otherwise a random string will be + ## created. Alternatively an existing secret can be provided. Note that postgres + ## DOES NOT tolerate a change of the admin password when upgrading. + # postgresqlPassword: + + ## Use an existing secret instead of creating a new one. It must have a + ## postgresql-password key containing the password for the postgres user. + # existingSecret: + + # Consider replication. These are the defaults for the basic settings. + # replicationUsername: repl_user + # replicationPassword: repl_password # generate a random password `openssl rand -hex 32` + + # image: + # repository: bitnami/postgresql + # tag: + + primary: + persistence: + ## We use the defaults here, but they will probably be modified for most deployments. + enabled: true + size: 8Gi + ## Provide an existing PersistentVolumeClaim to be reused. + # existingClaim: + + # Consider replication. These are the defaults for the basic settings. + readReplicas: + enabled: false + replicaCount: 1 - ## Use an existing secret instead of creating a new one. It must have a - ## postgresql-password key containing the password for the posgres user. - # existingSecret: - image: - repository: bitnami/postgresql - tag: 12.8.0 - persistence: - ## We use the defaults here, but they will probably be modified for most deployments. - enabled: true - size: 8Gi - ## Provide an existing PersistentVolumeClaim to be reused. - # existingClaim: - # Consider replication. These are the defaults for the basic settings. - replication: - enabled: false - user: repl_user - password: repl_password # generate a random password `openssl rand -hex 32` - slaveReplicas: 1 redis: # If set to true, a HA redis will be included in the Renku release. install: true @@ -854,7 +861,8 @@ dlf-chart: enabled: false dataset-operator-chart: enabled: true -csi-rclone: {} +csi-rclone: + {} # This section is only relevant if you are installing csi-rclone as part of Renku ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster. # storageClassName: csi-rclone diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml index 393c415139..c630909dbf 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_dump.yaml @@ -27,7 +27,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password ports: - containerPort: 5432 volumeMounts: @@ -81,7 +81,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml index d4395019f2..1ad02e0d18 100644 --- a/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml +++ b/helm-chart/utils/postgres_migrations/version_upgrades/psql_load.yaml @@ -28,7 +28,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /bitnami/postgresql name: pg-vol-new @@ -80,7 +80,7 @@ spec: valueFrom: secretKeyRef: name: renku-postgresql ## EDIT(optional) - most likely -postgresql - key: postgresql-password + key: postgres-password volumeMounts: - mountPath: /psql-dump-data/ name: pg-vol-tmp diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index 41fdf5ee1d..ba173ceff2 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,52 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.49.0 + +The PostgreSQL chart dependency has been upgraded, which requires modification of the postgres data volume of existing deployments. See [these instructions](https://github.com/SwissDataScienceCenter/renku/tree/master/helm-chart/utils/postgres_migrations/version_upgrades/README.md) for more details. + +* NEW/EDIT - *postgresql.persistence.existingClaim*: Renku `0.xx.0` upgrades the postgres chart dependency, which requires modification of the postgres data volume of existing deployments. See [these instructions](https://github.com/SwissDataScienceCenter/renku/tree/master/helm-chart/utils/postgres_migrations/version_upgrades/README.md) + +* EDIT - *postgresql*: The upgrade of the postgres chart dependency requires some restructuring of the postgres subchart values to match those of bitnami/postgresql chart version 14.0.1, namely: + +Old + ``` + postgresql: + postgresqlDatabase: + postgresqlUsername: + postgresqlPassword: + existingSecret: + persistence: + enabled: + size: + existingClaim: + replication: + enabled: + user: + password: + slaveReplicas: + ``` +New + ``` + postgresql: + auth: + username: + database: + postgresqlPassword: + existingSecret: + replicationUsername: + replicationPassword: + primary: + persistence: + enabled: + size: + existingClaim: + readReplicas: + enabled: + replicaCount: + ``` + +<<<<<<< HEAD ## Upgrading to Renku 0.48.1 The handling of privacy policy and terms of service content has been fine tuned. @@ -16,6 +62,8 @@ The handling of privacy policy and terms of service content has been fine tuned. * NEW ``ui.client.privacy.page.termsContent`` to customize the content of the Terms of Use page (supports Markdown). +======= +>>>>>>> 4a49d877 (chore: update values changelog) ## Upgrading to Renku 0.48.0 The handling of privacy policy and terms of service content has been slightly changed to make @@ -27,7 +75,6 @@ it more flexible. * NEW `ui.client.privacy.page.configMapPolicyKey` the key in the ConfigMap where the content for the privacy policy is located. * NEW `ui.client.privacy.page.configMapTermsKey` the key in the ConfigMap where the content for the terms of use is located. - ## Upgrading to Renku 0.47.0 We completely overhauled how mounting cloud storage in sessions works, relying on a new CSI driver based on RClone