diff --git a/.github/workflows/check-acceptance-test-code.yml b/.github/workflows/check-acceptance-test-code.yml index f272a5f77a..f50824df7f 100644 --- a/.github/workflows/check-acceptance-test-code.yml +++ b/.github/workflows/check-acceptance-test-code.yml @@ -8,7 +8,7 @@ jobs: name: Scala dependencies and code check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: run test compile run: | cd acceptance-tests diff --git a/.github/workflows/check-acceptance-test-fmt.yml b/.github/workflows/check-acceptance-test-fmt.yml index be1dbdeadf..b69bded5ce 100644 --- a/.github/workflows/check-acceptance-test-fmt.yml +++ b/.github/workflows/check-acceptance-test-fmt.yml @@ -8,7 +8,7 @@ jobs: name: Scala formatting check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: run scalafmt run: | cd acceptance-tests diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml index 6f0e127148..45e15469c2 100644 --- a/.github/workflows/create-release-branch.yml +++ b/.github/workflows/create-release-branch.yml @@ -16,7 +16,7 @@ jobs: create-release-pr: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}" @@ -42,9 +42,9 @@ jobs: git commit -m "chore: create release ${{ github.event.inputs.version }}" git push - name: Create Pull Request - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: - token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} + github-token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} script: | const { repo, owner } = context.repo; const result = await github.rest.pulls.create({ @@ -54,9 +54,11 @@ jobs: head: 'release-${{ github.event.inputs.version }}', base: '${{ github.event.inputs.target_branch }}', body: [ + 'Release ${{ github.event.inputs.version }}', + '', + 'This PR is auto-generated by [actions/github-script](https://github.com/actions/github-script).', + '', '/deploy ', - 'This PR is auto-generated by', - '[actions/github-script](https://github.com/actions/github-script).' ].join('\n') }); github.rest.issues.addLabels({ diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index 7a7d01e6ae..a5a233c429 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/generate-values-script.yaml b/.github/workflows/generate-values-script.yaml index dbc49a6160..3477797c07 100644 --- a/.github/workflows/generate-values-script.yaml +++ b/.github/workflows/generate-values-script.yaml @@ -15,7 +15,7 @@ jobs: - os: macos-11 - os: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: actions/setup-python@v5 @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-20.04 needs: [test-script] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: Azure/docker-login@v1 diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 8be7afa896..873d5b85a6 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -9,13 +9,13 @@ jobs: publish-chart: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.10.0 env: CHART_DIR: helm-chart/ CHART_NAME: renku @@ -45,11 +45,11 @@ jobs: needs: - "publish-chart" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - name: Rollout renku version - uses: SwissDataScienceCenter/renku-actions/rollout-renku-version@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/rollout-renku-version@v1.10.0 env: CHART_VERSION: ${{ needs.publish-chart.outputs.chart-version }} GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index c424e89dde..7ebd9810b0 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -14,7 +14,7 @@ jobs: publish-chart: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: azure/setup-helm@v3 @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.10.0 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index 9d9731aaa6..9ef9a87571 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -18,11 +18,11 @@ jobs: if: github.event.action != 'closed' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '17' + distribution: "temurin" + java-version: "17" - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -60,9 +60,9 @@ jobs: test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.10.0 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -73,11 +73,12 @@ jobs: runs-on: ubuntu-22.04 environment: name: ci-renku-${{ github.event.number }} + url: https://ci-renku-${{ github.event.number }}.dev.renku.ch steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.10.0 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -99,7 +100,7 @@ jobs: extra_values: "${{ needs.check-deploy.outputs.extra-values }}" - name: Check existing renkubot comment if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: peter-evans/find-comment@v2 + uses: peter-evans/find-comment@v3 id: findcomment with: issue-number: ${{ github.event.pull_request.number }} @@ -107,7 +108,7 @@ jobs: - name: Create comment pre deploy if: ${{ steps.findcomment.outputs.comment-id == 0 && needs.check-deploy.outputs.pr-contains-string == 'true' }} - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} issue-number: ${{ github.event.pull_request.number }} @@ -119,7 +120,7 @@ jobs: needs: [check-deploy, deploy-pr] runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -136,18 +137,19 @@ jobs: fail-fast: false max-parallel: 1 matrix: - tests: [ - publicProject, - privateProject, - updateProjects, - testDatasets, - useSession, - checkWorkflows, - rstudioSession - ] + tests: + [ + publicProject, + privateProject, + updateProjects, + testDatasets, + useSession, + checkWorkflows, + rstudioSession, + ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.10.0 if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -160,7 +162,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index 60b6b83e1c..9f922efd99 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku @@ -22,7 +22,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - uses: cypress-io/github-action@v5 id: cypress env: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cfe601c94c..12d757c3f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,89 @@ .. _changelog: +0.47.1 +------ + +This release only includes changes to the documentation and updates to the acceptance tests. +It doesn't bring any new features or bug fixes. + +0.47.0 +------ + +This release expands Renku's cloud storage functionality in two key ways: First, mounted storages +are now read **and write**, so you can use mounted storage as an active workspace for your data in a RenkuLab +session. Second, we have expanded the cloud storage services you can integrate with RenkuLab. You can now +mount not only S3 buckets, but also WebDAV-based storages and Azure Blobs. + +If you use SSH sessions via the CLI, you can use cloud storage there too! Configure cloud storage for your +project on RenkuLab.io, and those storages will be mounted in your remote session. Support for cloud +storage in local Renku sessions is still on our roadmap. + +This release also adds the ability to change which resource class your session uses when you unpause the +session, in case the original resource class is now full. + +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**🌟 New Features** + +- 🖋 **Notebooks,Data Services,CSI**: Support for read and write storage mounting in sessions using a new rclone based storage driver + (`#1707 `_, + `#92 `_, + `#1 `_). +- 🔌 **UI**: add support for more storage services + (`#2908 `_, + `#2915 `_). + +**✨ Improvements** + +- 🖌️ **UI**: Improve the look and feel of the home page + (`#2968 `_, + `#2937 `_, + `#2927 `_). +- 🔐 **UI**: Use password fields for credentials + (`#2920 `_). +- 🔧 **UI**: Allow users to modify non running sessions + (`#2942 `_). +- 🛑 **UI**: Improve feedback when starting sessions on outdated projects + (`#2985 `_). +- 🖌️ **UI**: Update the Renku logo and Renku browser icons + (`#2848 `_). + +**🐞 Bug Fixes** + +- **UI**: Resize the feedback badge on the session settings page + (`#2953 `_). +- **UI**: Fix the environment dropdown on the Start session page + (`#2949 `_). +- **UI**: Improve string validation when trying to upload a dataset file by URL + (`#2834 `_). + +Internal Changes +~~~~~~~~~~~~~~~~ + +**Improvements** + +- **UI**: RenkuLab admins can now add tolerations and node affinities to resource classes + (`#2916 `_). +- **UI**: RenkuLab admins can add multiple users to a resource pool at once via a list of emails + (`#2910 `_). +- **UI**: Use the renku-core API for session options + (`#2947 `_). +- **UI**: Specify a branch every time a renku-core API is invoked + (`#2977 `_). + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `amalthea 0.11.0 `_ +- `csi-rclone 0.1.5 `_ +- `renku-data-services 0.4.0 `_ +- `renku-notebooks 1.21.0 `_ +- `renku-ui 3.18.0 `_ +- `renku-ui 3.18.1 `_ +- `renku-ui 3.19.0 `_ + + 0.46.0 ------ @@ -13,7 +97,7 @@ User-Facing Changes - **UI**: Improve feedback when starting sessions on outdated projects (`#2985 `_). - **CLI**: Allow specifying storage to mount when launching Renkulab sessions from the CLI - (`#3629 _`). + (`#3629 `_). - **KG**: Remove the Free-Text Dataset Search API as improved functionality is offered by the Entities Search. (`#1833 `_). - **KG**: Add support for specifying ``templateRef`` and ``templateParameters`` on the Project Create API. @@ -25,7 +109,7 @@ User-Facing Changes (`#3687 `_). Internal Changes ----------------- +~~~~~~~~~~~~~~~~ **✨ Improvements** @@ -45,13 +129,13 @@ Individual Components ------ User-Facing Changes -------------------- +~~~~~~~~~~~~~~~~~~~ - **Core Service**: Removed support for metadata v9 projects in the UI. Migration to v10 is now required. - **Core Service**: Fixed a bug where projects weren't cloned shallowly, leading to large projects not working properly on the platform. Internal Changes ----------------- +~~~~~~~~~~~~~~~~ **🐞 Bug Fixes** @@ -78,7 +162,7 @@ are paused and resumed which caused resumed session to not be able to push to Gi or also it caused some sessions to not be able to resume after they have been paused. Internal Changes ----------------- +~~~~~~~~~~~~~~~~ **🐞 Bug Fixes** diff --git a/RELEASE.md b/RELEASE.md index e552b2ef0f..08b6caeee1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,8 +12,9 @@ This procedure should be followed for *any* release: * Create a release branch (e.g. `0.46.x`), if one does not already exist, with the [release action](https://github.com/SwissDataScienceCenter/renku/actions/workflows/create-release-branch.yml). * Create a `CHANGELOG` entry for the release and open a PR; create a deployment, this is the reference for the release. -* Note that any PR that should go into the release needs to target the release branch _not_ `master`. +* Note that any PR that should go into the release needs to target the release branch _not_ `master`. * All release branches should be protected. +* Use the "Rebase and Merge" button to merge release branches into `master`; do not squash commits. Acceptance tests have to pass on all release branches before merging. diff --git a/acceptance-tests/build.sbt b/acceptance-tests/build.sbt index c2d9473617..a3d00290a8 100644 --- a/acceptance-tests/build.sbt +++ b/acceptance-tests/build.sbt @@ -32,7 +32,7 @@ val circeVersion = "0.14.6" libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.14" libraryDependencies += "com.github.pureconfig" %% "pureconfig" % "0.17.5" % Test -libraryDependencies += "eu.timepit" %% "refined" % "0.11.0" % Test +libraryDependencies += "eu.timepit" %% "refined" % "0.11.1" % Test libraryDependencies += "io.circe" %% "circe-core" % circeVersion % Test libraryDependencies += "io.circe" %% "circe-literal" % circeVersion % Test libraryDependencies += "io.circe" %% "circe-parser" % circeVersion % Test @@ -40,10 +40,10 @@ libraryDependencies += "io.circe" %% "circe-optics" libraryDependencies += "org.http4s" %% "http4s-blaze-client" % "0.23.16" % Test libraryDependencies += "org.http4s" %% "http4s-circe" % "0.23.25" % Test libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.17.0" % Test -libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.17" % Test +libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.18" % Test libraryDependencies += "org.scalatestplus" %% "selenium-4-1" % "3.2.12.1" % Test libraryDependencies += "org.seleniumhq.selenium" % "selenium-http-jdk-client" % "4.13.0" % Test -libraryDependencies += "org.seleniumhq.selenium" % "selenium-java" % "4.16.1" % Test +libraryDependencies += "org.seleniumhq.selenium" % "selenium-java" % "4.17.0" % Test libraryDependencies += "org.slf4j" % "slf4j-log4j12" % "2.0.11" % Test libraryDependencies += "org.typelevel" %% "cats-effect" % "3.5.3" % Test libraryDependencies += "org.typelevel" %% "cats-effect-testing-scalatest" % "1.5.0" % Test diff --git a/cypress-tests/cypress/e2e/useSession.cy.ts b/cypress-tests/cypress/e2e/useSession.cy.ts index 1a3f0cff76..6348d10584 100644 --- a/cypress-tests/cypress/e2e/useSession.cy.ts +++ b/cypress-tests/cypress/e2e/useSession.cy.ts @@ -224,108 +224,135 @@ describe("Basic public project functionality", () => { }); it("Start a new session with cloud storage attached.", () => { - cy.stopAllSessionsForProject(projectIdentifier); - - cy.intercept("/ui-server/api/data/storage*").as("getProjectCloudStorage"); - - cy.getProjectSection("Settings").click(); - cy.getDataCy("settings-navbar") - .contains("a.nav-link", "Cloud Storage") - .should("be.visible") - .click(); - - // Add a S3 storage configuration if it doesn't exist - cy.wait("@getProjectCloudStorage").then(({ response }) => { - const storages = response.body as { storage: { name: string } }[]; - if (storages.find(({ storage }) => storage.name === "data_s3")) { + // verify that cloud storage is supported in the deployment + cy.request({ url: "ui-server/api/notebooks/version" }).then((resp) => { + if (!resp.body.versions[0].data.cloudstorageEnabled) { + cy.log( + "Skipping the test since the deployment doesn't support Cloud Storage" + ); return; } - cy.getDataCy("settings-container") - .find("button") - .contains("Add Cloud Storage") - .should("be.visible") - .click(); - - cy.get(".modal").contains("Add Cloud Storage").should("be.visible"); - - cy.get("label").contains("Name").click(); - cy.get(":focused").type("data_s3"); - - cy.get(".modal") - .contains("For AWS S3 buckets, supported URLs are of the form") - .should("be.visible"); - cy.get("label").contains("Endpoint URL").click(); - cy.get(":focused").type("s3://giab"); - - cy.get("label") - .contains("Requires credentials") - .siblings("input") - .click() - .should("not.be.checked"); - - // NOTE: Temporarily removed this from the tests until we enable read-write - // cy.get("label") - // .contains("Read-only") - // .siblings("input") - // .should("be.checked"); + cy.stopAllSessionsForProject(projectIdentifier); + cy.intercept("/ui-server/api/data/storage*").as("getProjectCloudStorage"); - cy.get("button[type='submit']") - .contains("Add Storage") + cy.getProjectSection("Settings").click(); + cy.getDataCy("settings-navbar") + .contains("a.nav-link", "Cloud Storage") .should("be.visible") .click(); - }); - - cy.getDataCy("more-menu").should("be.visible").click(); - cy.getProjectPageLink(projectIdentifier, "sessions/new") - .should("be.visible") - .first() - .click(); - // Wait for the image to be ready and start a session - cy.get(".renku-container") - .contains("A session gives you an environment") - .should("exist"); - cy.get(".renku-container .badge.bg-success", { timeout: TIMEOUTS.vlong }) - .contains("available") - .should("exist"); - cy.get(".renku-container button.btn-secondary", { timeout: TIMEOUTS.long }) - .contains("Start Session") - .should("exist") - .click(); - cy.get(".progress-box .progress-title").should("exist"); //.contains("Step 2 of 2"); - cy.get("button") - .contains(projectTestConfig.projectName) - .should("be.visible"); - cy.get(".progress-box .progress-title") - .contains("Starting Session") - .should("exist"); - cy.get(".progress-box .progress-title", { timeout: TIMEOUTS.vlong }).should( - "not.exist" - ); - - // Verify that the S3 data is mounted - cy.getIframe("iframe#session-iframe").within(() => { - cy.get(".jp-DirListing-content", { timeout: TIMEOUTS.long }).should( - "be.visible" - ); - cy.get(".jp-DirListing-item") - .contains("data_s3") - .should("be.visible") - .dblclick(); - - cy.get(".jp-DirListing-item") - .contains("README.s3_structure") + // Add a S3 storage configuration if it doesn't exist + cy.wait("@getProjectCloudStorage").then(({ response }) => { + const storages = response.body as { storage: { name: string } }[]; + if (storages.find(({ storage }) => storage.name === "data_s3")) { + return; + } + + cy.getDataCy("cloud-storage-section") + .find("button") + .contains("Add Cloud Storage") + .should("be.visible") + .click(); + cy.getDataCy("cloud-storage-edit-header") + .contains("Add Cloud Storage") + .should("be.visible"); + + cy.getDataCy("cloud-storage-edit-schema") + .contains("s3") + .should("be.visible") + .click(); + cy.getDataCy("cloud-storage-edit-providers") + .contains("AWS") + .should("be.visible") + .click(); + cy.getDataCy("cloud-storage-edit-next-button") + .should("be.visible") + .click(); + + cy.getDataCy("cloud-storage-edit-options").should("be.visible"); + cy.get("#sourcePath").should("have.value", "").type("giab"); + cy.get("#endpoint") + .should("have.value", "") + .type("http://s3.amazonaws.com"); + cy.getDataCy("cloud-storage-edit-next-button") + .should("be.visible") + .click(); + + cy.getDataCy("cloud-storage-edit-mount").should("be.visible"); + cy.get("#name").should("have.value", "").type("data_s3"); + cy.get("#mountPoint") + .should("have.value", "external_storage/data_s3") + .type("{selectAll}data_s3"); + cy.get("#readOnly").should("not.be.checked").check(); + + cy.getDataCy("cloud-storage-edit-update-button") + .should("be.visible") + .contains("Add") + .click(); + + cy.getDataCy("cloud-storage-edit-body").contains( + "storage data_s3 has been succesfully added" + ); + cy.getDataCy("cloud-storage-edit-close-button") + .should("be.visible") + .click(); + }); + + cy.getDataCy("more-menu").should("be.visible").click(); + cy.getProjectPageLink(projectIdentifier, "sessions/new") .should("be.visible") - .dblclick(); - - cy.get(".jp-FileEditor", { timeout: TIMEOUTS.long }).should("be.visible"); - cy.get(".jp-FileEditor") - .contains("The GIAB s3 bucket and URLs") + .first() + .click(); + + // Wait for the image to be ready and start a session + cy.get(".renku-container") + .contains("A session gives you an environment") + .should("exist"); + cy.get(".renku-container .badge.bg-success", { timeout: TIMEOUTS.vlong }) + .contains("available") + .should("exist"); + cy.getDataCy("cloud-storage-item").contains("data_s3").should("exist"); + cy.get("#cloud-storage-data_s3-active").should("be.checked"); + cy.get(".renku-container button.btn-secondary", { timeout: TIMEOUTS.long }) + .contains("Start Session") + .should("exist") + .click(); + cy.get(".progress-box .progress-title").should("exist"); //.contains("Step 2 of 2"); + cy.get("button") + .contains(projectTestConfig.projectName) .should("be.visible"); + cy.get(".progress-box .progress-title") + .contains("Starting Session") + .should("exist"); + cy.get(".progress-box .progress-title", { timeout: TIMEOUTS.vlong }).should( + "not.exist" + ); + + // Verify that the S3 data is mounted + cy.getIframe("iframe#session-iframe").within(() => { + cy.get(".jp-DirListing-content", { timeout: TIMEOUTS.long }).should( + "be.visible" + ); + cy.get(".jp-DirListing-item") + .contains("data_s3") + .should("be.visible") + .dblclick(); + + cy.get(".jp-DirListing-item") + .contains("README.s3_structure") + .should("be.visible") + .dblclick(); + + cy.get(".jp-FileEditor", { timeout: TIMEOUTS.long }).should("be.visible"); + cy.get(".jp-FileEditor") + .contains("The GIAB s3 bucket and URLs") + .should("be.visible"); + }); + + cy.pauseSession(); + cy.deleteSession(); }); - - cy.pauseSession(); - cy.deleteSession(); }); + }); diff --git a/docs/_static/images/cloud_storage_1.png b/docs/_static/images/cloud_storage_1.png new file mode 100644 index 0000000000..38b7944145 Binary files /dev/null and b/docs/_static/images/cloud_storage_1.png differ diff --git a/docs/_static/images/cloud_storage_10.png b/docs/_static/images/cloud_storage_10.png new file mode 100644 index 0000000000..c8654ceb34 Binary files /dev/null and b/docs/_static/images/cloud_storage_10.png differ diff --git a/docs/_static/images/cloud_storage_11.png b/docs/_static/images/cloud_storage_11.png new file mode 100644 index 0000000000..ccb906b4c3 Binary files /dev/null and b/docs/_static/images/cloud_storage_11.png differ diff --git a/docs/_static/images/cloud_storage_2.png b/docs/_static/images/cloud_storage_2.png new file mode 100644 index 0000000000..80392abc0b Binary files /dev/null and b/docs/_static/images/cloud_storage_2.png differ diff --git a/docs/_static/images/cloud_storage_3.png b/docs/_static/images/cloud_storage_3.png new file mode 100644 index 0000000000..a868959d17 Binary files /dev/null and b/docs/_static/images/cloud_storage_3.png differ diff --git a/docs/_static/images/cloud_storage_4.png b/docs/_static/images/cloud_storage_4.png new file mode 100644 index 0000000000..e062628e73 Binary files /dev/null and b/docs/_static/images/cloud_storage_4.png differ diff --git a/docs/_static/images/cloud_storage_5.png b/docs/_static/images/cloud_storage_5.png new file mode 100644 index 0000000000..1a335f71e4 Binary files /dev/null and b/docs/_static/images/cloud_storage_5.png differ diff --git a/docs/_static/images/cloud_storage_6.png b/docs/_static/images/cloud_storage_6.png new file mode 100644 index 0000000000..a5fa7cc325 Binary files /dev/null and b/docs/_static/images/cloud_storage_6.png differ diff --git a/docs/_static/images/cloud_storage_7.png b/docs/_static/images/cloud_storage_7.png new file mode 100644 index 0000000000..1fb8907e6f Binary files /dev/null and b/docs/_static/images/cloud_storage_7.png differ diff --git a/docs/_static/images/cloud_storage_8.png b/docs/_static/images/cloud_storage_8.png new file mode 100644 index 0000000000..2a4deaa055 Binary files /dev/null and b/docs/_static/images/cloud_storage_8.png differ diff --git a/docs/_static/images/cloud_storage_9.png b/docs/_static/images/cloud_storage_9.png new file mode 100644 index 0000000000..10eb7aefd3 Binary files /dev/null and b/docs/_static/images/cloud_storage_9.png differ diff --git a/docs/how-to-guides/admin/sessions-cloud-storage.rst b/docs/how-to-guides/admin/sessions-cloud-storage.rst index b02a01565f..5ada83b8a0 100644 --- a/docs/how-to-guides/admin/sessions-cloud-storage.rst +++ b/docs/how-to-guides/admin/sessions-cloud-storage.rst @@ -3,17 +3,35 @@ Enable mounting cloud storage in users-sessions ----------------------------------------------- -Renkulab supports mounting cloud storage in users-sessions. -To enable the feature you need to set the following values in your `values-file.yaml`: +Renkulab supports mounting cloud storage in users-sessions. We use a CSI driver based on +`rclone `_. + +To enable the feature you need to configure it in your `values-file.yaml`: .. code-block:: yaml + # install the CSI driver + global: + csi-rclone: + install: true + notebooks: cloudstorage: - s3: - enabled: true - installDatashim: true + enabled: true + storageClass: csi-rclone + csi-rclone: + csiNodepluginRclone: + # optional tolerations if using node pools with taints + tolerations: + - key: renku.io/dedicated + operator: Equal + value: user + effect: NoSchedule + - key: renku.io/gpu + operator: Equal + value: "true" + effect: NoSchedule -The `installDatashim` option will deploy the Datashim Helm chart in the same -namespace as the `Renkulab` chart. +The ``global.csi-rclone.install`` option will install the ``csi-rclone`` Helm chart in the same namespace +as the renku chart. \ No newline at end of file diff --git a/docs/how-to-guides/renkulab/external-storage.rst b/docs/how-to-guides/renkulab/external-storage.rst new file mode 100644 index 0000000000..3c7bcac5be --- /dev/null +++ b/docs/how-to-guides/renkulab/external-storage.rst @@ -0,0 +1,235 @@ +.. _external_storage: + +External Storage in Renku Sessions +================================== + +Renku supports accessing data from external storage services like +S3, Azure Blob, WebDAV, etc. in interactive sessions. Access to external +storage can be configured via project settings. + +In the sections below we describe how to add and manage storage configuration to your projects +and highlight some current limitations: + +.. contents:: + :local: + +.. note:: + + Note that this feature is optional; it is enabled at + `renkulab.io `_ but other Renku deployments might not + support it. Contact your administrator for more details. + +.. warning:: + + We support both Read-only and Read/Write access modes. + Please note that explicitly marking an attached storage as Read-only is + **strongly** encouraged every time you don't have write permissions. This + prevents unexpected error messages when trying to access the storage. Furthermore, + please be aware that allowing write access to your data could result in data loss, data corruption or + data being overwritten. Consult the documentation of your cloud storage provider to learn how to + setup access controls and data versioning if you want to protect your data. + +Adding external storage to a project +------------------------------------ + +Currently, we support configuring external storage access per project. +Under the project's ``Settings`` tab, you can find the ``Cloud Storage`` +section. From there, you can click on the ``Add Cloud Storage`` button and follow the +instructions to go through the guided procedure. + +.. image:: ../../_static/images/cloud_storage_1.png + :align: center + :alt: Cloud Storage section in the project settings + +| + +Step-by-step mode +~~~~~~~~~~~~~~~~~ + +This is the default mode. It will guide you through the process of adding an +external storage to use in your sessions. + +Depending on the storage service, the list of options might be long, especially +when clicking the "Show full list" switch. Most of the time, the basic options +are sufficient. For more information, please refer to either +the `Rclone documentation `_ or the documentation from +the storage provider you are using. + + +Advanced Mode +~~~~~~~~~~~~~ + +If you are familiar with `Rclone `_, you can switch to +the "Advanced mode" and copy/paste from any Rclone configuration file. + +Mind that we require some additional information that cannot be inferred from +the configuration file, such as the storage name, the access mode, and the mount +point for the sessions. + + +Specific storage providers +-------------------------- + +The backend supports a variety of storage providers. It is based on `Rclone `_ +and we therefore support `most` of the storage services that Rclone supports, except for +anything requiring the OAuth flow or virtual rclone providers. The experience is optimized for the +most commonly used services, such as S3 and WebDAV. Please let us know if you require access to another +service that is not covered by the currently-available options! + +S3 +~~ + +Here is an example of attaching an S3 bucket to your project. +We pick the public dataset +`Genome in a Bottle (GIAB) `_ +hosted on AWS and show how to attach it to a Renku interactive session: + +- Navigate to the ``Settings: Cloud storage`` in one of your Renku projects. + +- Click on the ``Add Cloud Storage`` button and select ``S3`` for the ``Storage Type``, + then pick ``AWS`` on the provider list. Click on the ``Next`` button at the bottom. + + .. image:: ../../_static/images/cloud_storage_2.png + :align: center + :alt: Storage type and provider selection + +| + +- Fill in the form by using ``giab`` as the ``Source path``, and + ``http://s3.amazonaws.com`` as the endpoint. We don't need any credentials, and + the region is optional. Click on the ``"Next`` button. + + .. image:: ../../_static/images/cloud_storage_3.png + :align: center + :alt: GIAB bucket options configuration + +| + +- On the last page, pick any name for the storage (e.g. ``Giab``), then mark + this as Read-only. You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the ``Add storage`` button. + + .. image:: ../../_static/images/cloud_storage_4.png + :align: center + :alt: GIAB bucket final steps + +| + +.. note:: + The ``mount point`` is relative to the root of the project path in the session. + +You can now start a new session and access the ``giab`` bucket at the ``external_storage/giab`` path in your project. + + +Azure Blob +~~~~~~~~~~ + +For this example, we host some data files in the folder ``test-data`` +hosted on Azure and show how to attach it to a Renku interactive session: + +- Navigate to the ``Settings - Cloud storage`` in your project. + +- Click on the ``Add Cloud Storage`` button and select ``azureblob`` for the + ``Storage Type``. Click on the ``Next`` button at the bottom. + + .. image:: ../../_static/images/cloud_storage_5.png + :align: center + :alt: Azure blob storage selection + +| + +- Fill in the form by filling the ``Source path`` to mount (keep it blank to + mount the default root, or specify the folder) and the Account Name. Fill + in the ``Shared Key``. Click on the ``Next`` button. + + .. image:: ../../_static/images/cloud_storage_6.png + :align: center + :alt: Azure blob options configuration + +| + +- On the last page, pick any name for the storage (e.g: `blob_storage``). + You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the ``Add storage`` button. + + .. image:: ../../_static/images/cloud_storage_7.png + :align: center + :alt: Azure blob final steps + +| + +- When you start your session, the blob storage will be available at the specified mount point. + +WebDAV +~~~~~~ + +Many storage services, offer access via the WebDAV protocol. Renku supports +mounting WebDAV resources in interactive sessions. + +.. warning:: + We strongly encourage you to generate a token for accessing your private + OwnCloud/Nextcloud WebDAV shares to avoid sharing your account password. To create the token, go to + "Settings: Security" in your cloud storage application and generate a new + "Application pass-code" at the bottom of the page. + +Here we show an example of adding an `ETH Polybox `_ drive to +the session, but the procedure will be similar for any storage accessible via WebDAV. + +- Navigate to ``Settings - Cloud storage`` in your Renku project. +- Click on the "Add Cloud Storage" button and select ``webdav`` for + the "Storage Type". Click on the "Next" button at the bottom. + + .. image:: ../../_static/images/cloud_storage_8.png + :align: center + :alt: WebDAV storage selection + +| + +- Fill in the form by providing the ``Source path`` to mount (keep it blank to mount the + default root, or specify the folder), and the URL to specify the host to connect to + (``https://polybox.ethz.ch/remote.php/webdav/`` for `ETH Polybox `_). + Fill in the ``Username`` and the ``Token (or password)``. Click on the "Next" button. + + .. image:: ../../_static/images/cloud_storage_9.png + :align: center + :alt: WebDAV options configuration + +| + +- On the last page, pick any name for the storage (e.g: ``webdab_storage``). + You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the "Add storage" button. + + .. image:: ../../_static/images/cloud_storage_10.png + :align: center + :alt: WebDAV final steps + +| + +- You will be able to access the cloud storage at the specified + mount point within your session. + + .. image:: ../../_static/images/cloud_storage_11.png + :align: center + :alt: Access to the mounted storage + +| + +Credentials +----------- + +For the time being, you cannot persist storage credentials in RenkuLab. If you need to +provide credentials to access storage, you will be asked to provide them +when starting a new session. The storage can be optionally disabled when launching a +session, if you do not wish to use it or if you do not have the required credentials. +If you pause and resume your session, you do not need to enter credentials again. + +Performance +----------- + +Performance varies based on the specific storage service, the access mode, +and the available bandwidth. Consider caching data locally if large amounts of +data need to be read or written repeatedly. diff --git a/docs/how-to-guides/renkulab/index.rst b/docs/how-to-guides/renkulab/index.rst index 407f2def81..137700f4d8 100644 --- a/docs/how-to-guides/renkulab/index.rst +++ b/docs/how-to-guides/renkulab/index.rst @@ -7,6 +7,6 @@ RenkuLab :maxdepth: 1 session-stopping-and-saving + external-storage ssh-into-sessions - s3 collaboration diff --git a/docs/how-to-guides/renkulab/s3.rst b/docs/how-to-guides/renkulab/s3.rst deleted file mode 100644 index 1ceed06345..0000000000 --- a/docs/how-to-guides/renkulab/s3.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _s3: - -Mount S3 Buckets in Renku Sessions -================================== - -Renku has an optional feature that allows users to mount and access data in -any S3-compatible object storage. This feature is enabled at `renkulab.io `_ -but please note that on other Renku deployment this may not be the case. -If your deployment does not have this feature enabled you should contact -your administrator. - -Amazon AWS initially came up with the S3 API for object storage. However, nowadays -almost all cloud providers have S3-compatible object storage. For example, Google Cloud Storage -is S3 compatible and can be used in Renku. Please refer to your cloud provider documentation -for more details. - -Step-by-step Instructions -------------------------- - -There are many public datasets that are hosted by AWS or other cloud providers on -S3-compatible object storage. One such dataset is the `Genome in a Bottle (GIAB) `_ -dataset hosted on AWS. We will use the GIAB dataset to demonstrate how to bring data in S3 to a Renku interactive session: - -1. Navigate to the ``Sessions`` tab in your Renku project. -2. Click the green ``New Session`` button on the right side of the page. -3. Click on the ``Do you want to select the branch, commit, image or configure cloud storage`` link. -4. Click on the ``Configure Cloud Storage`` button in the section that was revealed in the previous step. -5. In the form that opened, enter the ``Endpoint`` for the GIAB bucket as ``http://s3.amazonaws.com``. -6. The only other field you have to fill in is the ``Bucket Name`` field. For the GIAB bucket this is simply ``giab``. The ``Access Key`` and ``Secret Key`` fields can be left empty because this is a public bucket. -7. Click ``Save``. -8. Click ``Start session``. -9. Once you session is ready the bucket will be available inside your session at ``/cloudstorage/``. In this case for the GIAB bucket it will be available at ``/cloudstorage/giab``. - -Limitations ------------ - -There are a few limitations of this feature that users should be aware of. Please note that as we -further develop this feature some of these limitations will be relaxed or fully lifted. - -* **Accessing data in mounted buckets is potentially much slower than accessing data on disk.** - All content that is available in the bucket is fetched over the internet every time you need to - access it. Therefore, using this feature will be slower compared to when data is downloaded to disk. - However, the benefit is that mounting a bucket does not require any additional disk space. - So you could mount a bucket which has 1TB of data in your session and you would not need to request 1TB of - storage from Renku. We are investigating methods to provide caching or other means to improve the performance. -* **All mounted buckets have to have unique names.** There is no limit to how many buckets can be - mounted in a session but because all buckets are mounted in the same folder (i.e. ``/cloudstorage``) - if you try to mount two buckets with the same name then one mount would overwrite the other. - However, Renku will not let you get to this point and will prevent you from launching a session if - there are duplicate bucket names (regardless of the endpoint) across all the buckets you are trying - to mount. -* **Buckets can only be mounted in `read-only` mode** regardless of whether your credentials - (if provided) allow you to read and write in the bucket you are mounting. This is just a precaution - that may eventually be removed as the feature is further developed. diff --git a/docs/renku-python b/docs/renku-python index c1be009688..af89ce6ac1 160000 --- a/docs/renku-python +++ b/docs/renku-python @@ -1 +1 @@ -Subproject commit c1be009688b6a73bafa90f93896d96f31d438d1e +Subproject commit af89ce6ac12e00544158bacba87df82fdb03585f diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index ee038afaf3..7ca1b17ace 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -4,6 +4,7 @@ adhoc admin Amalthea analytics +api app aqs args @@ -249,6 +250,7 @@ statuspage stderr stdin stdout +storages subclass Subclass subclasses @@ -301,6 +303,7 @@ unmapped unmerged Unmount unmount +unpause unpushed unschedulable untracked diff --git a/docs/topic-guides/data/data.rst b/docs/topic-guides/data/data.rst index d2f37aa349..1197fa88b7 100644 --- a/docs/topic-guides/data/data.rst +++ b/docs/topic-guides/data/data.rst @@ -12,6 +12,12 @@ large files on the server and allows you to work with your repository without actually having a local copy of the data. You can pull the data from the server as it becomes needed, saving you time and resources. +However, git LFS is not efficient when working with a large amount of data. +Hence, we heavily recommend Renku users (especially when dealing with large amount of data) +to either store data files locally outside the Renku project folder in a +local filesystem, or even better +to provision :ref:`external_storage`, such as S3 or Azure Blob storage. + Using git LFS responsibly ------------------------- @@ -27,6 +33,17 @@ of data to each of their JupterLab sessions. Not only will this take a long time, but it might also eventually lead to resource starvation on the host node. + +Working with external cloud storage +----------------------------------- +If you setup external storage from the beginning of your project, you can more +easily share and handle your data. You need to control the access rights to +your external cloud storage data directly through your cloud storage provider. + + +The instructions to set up your cloud storage for a Renku project are in +:ref:`external_storage`. + Data in JupyterLab sessions --------------------------- diff --git a/event.json b/event.json deleted file mode 100644 index 446545887f..0000000000 --- a/event.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "action":"workflow_dispatch", - "inputs":{ - "version":"9.99.9" - } -} diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml index 16d42a37f7..f28bdc16d7 100644 --- a/helm-chart/renku/requirements.yaml +++ b/helm-chart/renku/requirements.yaml @@ -23,8 +23,13 @@ dependencies: alias: jena - name: amalthea repository: "https://swissdatasciencecenter.github.io/helm-charts/" - version: "0.10.0" + version: "0.11.0" - name: dlf-chart repository: "https://swissdatasciencecenter.github.io/datashim/" version: "0.3.9-renku-2" condition: notebooks.cloudstorage.s3.installDatashim +- name: csi-rclone + repository: "https://swissdatasciencecenter.github.io/helm-charts/" + version: "0.1.5" + condition: global.csi-rclone.install + diff --git a/helm-chart/renku/templates/notebooks/statefulset.yaml b/helm-chart/renku/templates/notebooks/statefulset.yaml index b537f101aa..7985c4c0e1 100644 --- a/helm-chart/renku/templates/notebooks/statefulset.yaml +++ b/helm-chart/renku/templates/notebooks/statefulset.yaml @@ -158,14 +158,10 @@ spec: {{- include "certificates.env.python" . | nindent 12 }} - name: NB_SESSIONS__ENFORCE_CPU_LIMITS value: {{ .Values.notebooks.enforceCPULimits | quote }} - - name: NB_CLOUD_STORAGE__S3__ENABLED - value: {{ .Values.notebooks.cloudstorage.s3.enabled | quote }} - - name: NB_CLOUD_STORAGE__S3__READ_ONLY - value: {{ .Values.notebooks.cloudstorage.s3.readOnly | quote }} - - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED - value: {{ .Values.notebooks.cloudstorage.azureBlob.enabled | quote }} - - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY - value: {{ .Values.notebooks.cloudstorage.azureBlob.readOnly | quote }} + - name: NB_CLOUD_STORAGE__ENABLED + value: {{ .Values.notebooks.cloudstorage.enabled | quote }} + - name: NB_CLOUD_STORAGE__STORAGE_CLASS + value: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS value: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} - name: NB_VERSION diff --git a/helm-chart/renku/templates/notebooks/test.yaml b/helm-chart/renku/templates/notebooks/test.yaml index 0cd7881162..8fa4de31cf 100644 --- a/helm-chart/renku/templates/notebooks/test.yaml +++ b/helm-chart/renku/templates/notebooks/test.yaml @@ -85,14 +85,10 @@ spec: {{- end }} - name: NB_SESSIONS__ENFORCE_CPU_LIMITS value: {{ $.Values.notebooks.enforceCPULimits | quote }} - - name: NB_CLOUD_STORAGE__S3__ENABLED - value: {{ $.Values.notebooks.cloudstorage.s3.enabled | quote }} - - name: NB_CLOUD_STORAGE__S3__READ_ONLY - value: {{ $.Values.notebooks.cloudstorage.s3.readOnly | quote }} - - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED - value: {{ $.Values.notebooks.cloudstorage.azureBlob.enabled | quote }} - - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY - value: {{ $.Values.notebooks.cloudstorage.azureBlob.readOnly | quote }} + - name: NB_CLOUD_STORAGE__ENABLED + value: {{ $.Values.notebooks.cloudstorage.enabled | quote }} + - name: NB_CLOUD_STORAGE__STORAGE_CLASS + value: {{ $.Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }} - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS value: {{ $.Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }} - name: NB_VERSION diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 288ed366d4..79a8dbefa0 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -186,6 +186,10 @@ global: ## The contents of the 'password' key is used. ## The secret is not re-generated or modified in any way if it already exists. passwordSecretName: renku-db-common-password + csi-rclone: + # Set to true to install csi-rclone alongside renku + # Make sure to configure the csi-rclone section below correctly in that case. + install: false ## Ingress configuration ## See: https://kubernetes.io/docs/concepts/services-networking/ingress/ ingress: @@ -609,7 +613,7 @@ ui: replicaCount: 1 image: repository: renku/renku-ui - tag: "3.17.3" + tag: "3.19.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -768,7 +772,7 @@ ui: keepCookies: [] image: repository: renku/renku-ui-server - tag: "3.17.3" + tag: "3.19.0" pullPolicy: IfNotPresent imagePullSecrets: [] nameOverride: "" @@ -837,10 +841,6 @@ amalthea: scope: clusterWide: false deployCrd: true - ## If enabling S3 mounts uncomment the section below - # extraChildResources: - # - name: datasets - # group: com.ie.ibm.hpsys ## Used only if S3 Mounts are enabled dlf-chart: csi-sidecars-rbac: @@ -854,20 +854,24 @@ dlf-chart: enabled: false dataset-operator-chart: enabled: true +csi-rclone: {} + # This section is only relevant if you are installing csi-rclone as part of Renku + ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster. + # storageClassName: csi-rclone + # csiNodepluginRclone: + # nodeSelector: {} + # Set tolerations if you have taints on your user session nodes. The csi has to run on every node + # where it is used. + # tolerations: [] + # affinity: {} notebooks: cloudstorage: - ## If enabled this will allow mounting of buckets into user sessions - ## from any S3-compatible storage provider. + ## If enabled this will allow mounting of rclone cloudstorage + enabled: false + ## should match csi-rclone.storageClassName if you use a custom name + # storageClass: csi-rclone s3: - enabled: false installDatashim: false - readOnly: true - ## Azure blob storage mounting support is experimental. The CSI - ## storage driver for mounting azure blob storage has to be already installed - ## in the cluster if enabled. - azureBlob: - enabled: false - readOnly: true # configuration for user session persistent volumes userSessionPersistentVolumes: enabled: true @@ -912,7 +916,7 @@ notebooks: targetCPUUtilizationPercentage: 50 image: repository: renku/renku-notebooks - tag: "1.20.3" + tag: "1.21.0" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -1030,15 +1034,15 @@ notebooks: gitRpcServer: image: name: renku/git-rpc-server - tag: "1.20.2" + tag: "1.21.0" gitHttpsProxy: image: name: renku/git-https-proxy - tag: "1.20.2" + tag: "1.21.0" gitClone: image: name: renku/git-clone - tag: "1.20.2" + tag: "1.21.0" service: type: ClusterIP port: 80 @@ -1091,12 +1095,12 @@ notebooks: sessionTypes: ["registered"] image: repository: renku/renku-notebooks-tests - tag: "1.20.2" + tag: "1.21.0" pullPolicy: IfNotPresent k8sWatcher: image: repository: renku/k8s-watcher - tag: "1.20.2" + tag: "1.21.0" pullPolicy: IfNotPresent resources: {} replicaCount: 1 @@ -1109,7 +1113,7 @@ notebooks: enabled: false image: repository: renku/ssh-jump-host - tag: "1.20.2" + tag: "1.21.0" pullPolicy: IfNotPresent resources: {} replicaCount: 1 @@ -1476,7 +1480,7 @@ initDb: dataService: image: repository: renku/renku-data-service - tag: "0.3.0" + tag: "0.4.0" pullPolicy: IfNotPresent keycloakSync: image: diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index ad0e9ea23e..1aa578b668 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,26 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.47.0 + +We completely overhauled how mounting cloud storage in sessions works, relying on a new CSI driver based on RClone +which has to be installed in the cluster for things to work. Either install it as part of Renku using the flag +mentioned below or install the csi-rclone chart manually and set the correct storage class in the values for the +notebooks service. + +* NEW `noteboks.cloudstorage.enabled` - set to `true` to enable mounting cloud storage in sessions. +* DELETE `notebooks.cloudstorage.s3.enabed` - superseeded by previous property. +* NEW `notebooks.cloudstorage.storageClass` - the storage class for the CSI Rclone chart, needed for new cloudstorage + to work. The default `csi-rclone` should be fine unless already in use. +* NEW `global.csi-rclone.install` - if `true` installs the csi-rclone chart alongside Renku. The chart is needed for + cloud storage in sessions to work. +* NEW `csi-rclone.storageClassName` - the storage class name the CSI drivers uses, should match what is configured in + the `storageClass` property mentioned above. +* NEW `csi-rclone.csiNodePlugin.tolerations` - Tolerations for the node plugin part of the CSI driver. Need to be set + in a way that allows it to be scheduled on user session nodes. By default this would mean `key=renku.io/dedicated`, + `operator=Equal`, `value=user` and `effect=NoSchedule` + + ## Upgrading to Renku 0.43.0 * DELETE `graph.gitlab.url` has been removed as graph services uses the `global.gitlab.url`.