feat: support for renku 2 sessions (#1810) (#1815)

Fixes #1800. See SwissDataScienceCenter/renku#3475. Add support for Renku 2.0 sessions: * Add a new endpoint `/v2/servers` for starting Renku 2.0 sessions * Renku 2.0 sessions can have any number of code repositories mounted from GitLab * Add new session annotation to describe Renku 2.0 sessions Co-authored-by: M. Alisafaee <[email protected]>
SwissDataScienceCenter · Mar 19, 2024 · 13fdd60 · 13fdd60
1 parent e326095
commit 13fdd60
Show file tree

Hide file tree

Showing 13 changed files with 496 additions and 131 deletions.
diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/general.py b/components/renku_data_services/notebooks/api/amalthea_patches/general.py
@@ -60,11 +60,11 @@ def session_affinity(server: "UserServer"):
     )
     preferred_match_expressions: List[Dict[str, str]] = []
     required_match_expressions: List[Dict[str, str]] = []
-    for affintiy in server.server_options.node_affinities:
-        if affintiy.required_during_scheduling:
-            required_match_expressions.append(affintiy.json_match_expression())
+    for affinity in server.server_options.node_affinities:
+        if affinity.required_during_scheduling:
+            required_match_expressions.append(affinity.json_match_expression())
         else:
-            preferred_match_expressions.append(affintiy.json_match_expression())
+            preferred_match_expressions.append(affinity.json_match_expression())
     return [
         {
             "type": "application/json-patch+json",
@@ -142,12 +142,12 @@ def test(server: "UserServer"):
     order of containers in the amalthea manifests is what the notebook service expects.
     """
     patches = []
-    # NOTE: Only the first 1 or 2 containers come "included" from Amalthea, the rest are patched in
-    # This tests checks whether the expected number and order is received from Amalthea and
+    # NOTE: Only the first 1 or 2 containers come "included" from Amalthea, the rest are patched
+    # in. This test checks whether the expected number and order is received from Amalthea and
     # does not use all containers.
     container_names = (
         config.sessions.containers.registered[:2]
-        if type(server._user) is RegisteredUser
+        if isinstance(server.user, RegisteredUser)
         else config.sessions.containers.anonymous[:1]
     )
     for container_ind, container_name in enumerate(container_names):
@@ -158,7 +158,7 @@ def test(server: "UserServer"):
                     {
                         "op": "test",
                         "path": (
-                            "/statefulset/spec/template/spec" f"/containers/{container_ind}/name"
+                            f"/statefulset/spec/template/spec/containers/{container_ind}/name"
                         ),
                         "value": container_name,
                     }
@@ -170,7 +170,7 @@ def test(server: "UserServer"):
 
 def oidc_unverified_email(server: "UserServer"):
     patches = []
-    if type(server._user) is RegisteredUser:
+    if isinstance(server.user, RegisteredUser):
         # modify oauth2 proxy to accept users whose email has not been verified
         # usually enabled for dev purposes
         patches.append(

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py b/components/renku_data_services/notebooks/api/amalthea_patches/git_proxy.py
@@ -1,7 +1,6 @@
 from typing import TYPE_CHECKING
 
 from ...config import config
-from ..classes.user import RegisteredUser
 from .utils import get_certificates_volume_mounts
 
 if TYPE_CHECKING:
@@ -15,6 +14,7 @@ def main(server: "UserServer"):
         read_only_etc_certs=True,
     )
     patches = []
+
     patches.append(
         {
             "type": "application/json-patch+json",
@@ -35,7 +35,7 @@ def main(server: "UserServer"):
                         "env": [
                             {
                                 "name": "REPOSITORY_URL",
-                                "value": server.gl_project.http_url_to_repo,
+                                "value": server.gl_project_url,
                             },
                             {
                                 "name": "GIT_PROXY_PORT",
@@ -47,19 +47,19 @@ def main(server: "UserServer"):
                             },
                             {
                                 "name": "GITLAB_OAUTH_TOKEN",
-                                "value": str(server._user.git_token),
+                                "value": str(server.user.git_token),
                             },
                             {
                                 "name": "GITLAB_OAUTH_TOKEN_EXPIRES_AT",
-                                "value": str(server._user.git_token_expires_at),
+                                "value": str(server.user.git_token_expires_at),
                             },
                             {
                                 "name": "RENKU_ACCESS_TOKEN",
-                                "value": str(server._user.access_token),
+                                "value": str(server.user.access_token),
                             },
                             {
                                 "name": "RENKU_REFRESH_TOKEN",
-                                "value": str(server._user.refresh_token),
+                                "value": str(server.user.refresh_token),
                             },
                             {
                                 "name": "RENKU_REALM",
@@ -79,9 +79,7 @@ def main(server: "UserServer"):
                             },
                             {
                                 "name": "ANONYMOUS_SESSION",
-                                "value": (
-                                    "false" if type(server._user) is RegisteredUser else "true"
-                                ),
+                                "value": "true" if server.user.anonymous else "false",
                             },
                         ],
                         "livenessProbe": {

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/git_sidecar.py b/components/renku_data_services/notebooks/api/amalthea_patches/git_sidecar.py
@@ -13,6 +13,8 @@ def main(server: "UserServer"):
     if not isinstance(server.user, RegisteredUser):
         return []
 
+    gl_project_path = server.gl_project_path or "."
+
     patches = [
         {
             "type": "application/json-patch+json",
@@ -36,7 +38,7 @@ def main(server: "UserServer"):
                         "env": [
                             {
                                 "name": "GIT_RPC_MOUNT_PATH",
-                                "value": f"/work/{server.gl_project.path}",
+                                "value": f"/work/{gl_project_path}",
                             },
                             {
                                 "name": "GIT_RPC_PORT",
@@ -92,9 +94,9 @@ def main(server: "UserServer"):
                         },
                         "volumeMounts": [
                             {
-                                "mountPath": f"/work/{server.gl_project.path}/",
+                                "mountPath": f"/work/{gl_project_path}/",
                                 "name": "workspace",
-                                "subPath": f"{server.gl_project.path}/",
+                                "subPath": f"{gl_project_path}",
                             }
                         ],
                         "livenessProbe": {

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py b/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py
@@ -1,11 +1,11 @@
+import json
 import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from kubernetes import client
 
 from ...config import config
-from ..classes.user import RegisteredUser
 from .utils import get_certificates_volume_mounts
 
 if TYPE_CHECKING:
@@ -18,11 +18,25 @@ def git_clone(server: "UserServer"):
         etc_certs=True,
         read_only_etc_certs=True,
     )
+
+    gl_project_path = server.gl_project_path or ""
+
     env = [
-        {"name": "GIT_CLONE_MOUNT_PATH", "value": server.work_dir.absolute().as_posix()},
+        {
+            "name": "GIT_CLONE_REPOSITORIES",
+            "value": json.dumps(server.repositories),
+        },
+        {
+            "name": "GIT_CLONE_WORKSPACE_MOUNT_PATH",
+            "value": server.workspace_mount_path.absolute().as_posix(),
+        },
         {
             "name": "GIT_CLONE_REPOSITORY_URL",
-            "value": server.gl_project.http_url_to_repo,
+            "value": server.gl_project.http_url_to_repo if server.gl_project else None,
+        },
+        {
+            "name": "GIT_CLONE_MOUNT_PATH",
+            "value": (server.workspace_mount_path / gl_project_path).absolute().as_posix(),
         },
         {
             "name": "GIT_CLONE_LFS_AUTO_FETCH",
@@ -62,7 +76,7 @@ def git_clone(server: "UserServer"):
             "value": str(Path(etc_cert_volume_mount[0]["mountPath"]) / "ca-certificates.crt"),
         },
     ]
-    if type(server.user) is RegisteredUser:
+    if not server.user.anonymous:
         env += [
             {"name": "GIT_CLONE_USER__EMAIL", "value": server.user.gitlab_user.email},
             {

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/jupyter_server.py b/components/renku_data_services/notebooks/api/amalthea_patches/jupyter_server.py
@@ -12,13 +12,15 @@
 
 def env(server: "UserServer"):
     # amalthea always makes the jupyter server the first container in the statefulset
+    gl_project_path = server.gl_project_path or ""
+
     patch_list = [
         {
             "op": "add",
             "path": "/statefulset/spec/template/spec/containers/0/env/-",
             "value": {
                 "name": "RENKU_USERNAME",
-                "value": server._user.username,
+                "value": server.user.username,
             },
         },
         {
@@ -41,7 +43,7 @@ def env(server: "UserServer"):
             # relative to $HOME.
             "value": {
                 "name": "MOUNT_PATH",
-                "value": f"/work/{server.gl_project.path}",
+                "value": f"/work/{gl_project_path}",
             },
         },
         {
@@ -106,8 +108,8 @@ def image_pull_secret(server: "UserServer"):
             "auths": {
                 config.git.registry: {
                     "Username": "oauth2",
-                    "Password": server._user.git_token,
-                    "Email": server._user.gitlab_user.email,
+                    "Password": server.user.git_token,
+                    "Email": server.user.gitlab_user.email,
                 }
             }
         }