Merge data connectors and notebooks/amalthea migration

.devcontainer/.poetry_cache/.keep

@@ -1 +0,0 @@
-

.devcontainer/devcontainer.json

@@ -11,21 +11,25 @@
 			"command": "poetry self add poetry-polylith-plugin"
 		},
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/mpriscella/features/kind:1": {},
 		"ghcr.io/devcontainers-contrib/features/gh-release:1": {
 			"repo": "authzed/zed",
-			"binaryNames": "zed"
+			"binaryNames": "zed",
+			"version": "v0.21.1"
 		},
 		"ghcr.io/devcontainers-contrib/features/spicedb:1": {},
 		"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
 			"minikube": "none"
 		},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {},
+		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
+			"jqVersion": "latest",
+			"yqVersion": "latest"
+		},
 		"ghcr.io/dhoeric/features/k9s:1": {},
 		"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
 			"command": "alias k=kubectl"
 		},
-		"ghcr.io/devcontainers-contrib/features/rclone:1": {}
+		"ghcr.io/devcontainers-contrib/features/rclone:1": {},
+		"./k3d": {}
 	},
 	"overrideFeatureInstallOrder": [
 		"ghcr.io/devcontainers-contrib/features/poetry",

.devcontainer/docker-compose.yml

@@ -23,6 +23,8 @@ services:
       ZED_TOKEN: renku
       ZED_INSECURE: "true"
       POETRY_CACHE_DIR: "/poetry_cache"
+      NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
+      NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
     network_mode: service:db
     depends_on:
       - db
@@ -43,6 +45,7 @@ services:
       - "8080:8080"
       - "5678:5678"
       - "50051:50051"
+      - "8888:80"
 
   swagger:
     image: swaggerapi/swagger-ui

.devcontainer/k3d/devcontainer-feature.json

@@ -0,0 +1,17 @@
+{
+    "id": "k3d",
+    "version": "1.0.0",
+    "name": "k3s based kubernetes cluster in docker",
+    "postCreateCommand": "k3d --version",
+    "installsAfter": [
+        "ghcr.io/devcontainers-contrib/features/bash-command"
+    ],
+    "options": {
+    "k3d_version": {
+      "type": "string",
+      "description": "k3d version to install",
+      "proposals": ["latest", "5.7.4"],
+      "default": "latest"
+    }
+  }
+}

.devcontainer/k3d/install.sh

@@ -0,0 +1,14 @@
+if [ "${K3D_VERSION}" != "none" ]; then
+    echo "Downloading k3d..."
+    if [ "${K3D_VERSION}" = "latest" ]; then
+        # Install and check the hash
+        curl -sSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+    else
+        find_version_from_git_tags K3D_VERSION https://github.com/kubernetes/K3D
+        if [ "${K3D_VERSION::1}" != "v" ]; then
+            K3D_VERSION="v${K3D_VERSION}"
+        fi
+        # Install and check the hash
+        curl -sSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG="${K3D_VERSION}" bash
+    fi
+fi

.github/workflows/acceptance-tests.yml

@@ -25,6 +25,8 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
+      amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
@@ -84,6 +86,8 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_data_services: "@${{ github.head_ref }}"
+          amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
+          amalthea_sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
 
   selenium-acceptance-tests:

.github/workflows/save_cache.yml

@@ -0,0 +1,41 @@
+name: Create cache from commits on main
+
+on:
+  push:
+    branches:
+      - main
+      - chore-add-kind
+  workflow_dispatch:
+
+
+jobs:
+  save-poetry-cache:
+    runs-on: ubuntu-latest
+    env:
+      CACHE_KEY: main-branch-poetry-cache-ubuntu
+      CACHE_PATH: .devcontainer/.poetry_cache
+      DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install python deps
+        uses: devcontainers/[email protected]
+        with:
+          runCmd: poetry install --with dev
+          push: always
+          skipContainerUserIdUpdate: false
+          imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+          cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+      - uses: actions/cache/save@v3
+        name: Create cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}

.github/workflows/test_publish.yml

@@ -70,6 +70,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -90,6 +95,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -111,6 +121,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -155,6 +170,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |

.pre-commit-config.yaml

@@ -18,8 +18,10 @@ repos:
       - id: check-toml
       - id: debug-statements
       - id: end-of-file-fixer
+        exclude: 'components/renku_data_services/message_queue/(avro_models|schemas)'
       - id: mixed-line-ending
       - id: trailing-whitespace
+        exclude: 'components/renku_data_services/message_queue/(avro_models|schemas)'
   - repo: https://github.com/asottile/yesqa
     rev: v1.5.0
     hooks:

Makefile

@@ -1,7 +1,7 @@
-.PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro kind_cluster install_amaltheas all
+.PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro k3d_cluster install_amaltheas all
 
-AMALTHEA_JS_VERSION ?= 0.11.0
-AMALTHEA_SESSIONS_VERSION ?= 0.0.1-new-operator-chart
+AMALTHEA_JS_VERSION ?= 0.12.2
+AMALTHEA_SESSIONS_VERSION ?= 0.0.10-new-operator-chart
 codegen_params = --input-file-type openapi --output-model-type pydantic_v2.BaseModel --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --set-default-enum-member --openapi-scopes schemas paths parameters --set-default-enum-member --use-one-literal-as-default --use-default
 
 define test_apispec_up_to_date
@@ -38,10 +38,23 @@ components/renku_data_services/notebooks/apispec.py: components/renku_data_servi
 	poetry run datamodel-codegen --input components/renku_data_services/notebooks/api.spec.yaml --output components/renku_data_services/notebooks/apispec.py --base-class renku_data_services.notebooks.apispec_base.BaseAPISpec $(codegen_params)
 components/renku_data_services/platform/apispec.py: components/renku_data_services/platform/api.spec.yaml
 	poetry run datamodel-codegen --input components/renku_data_services/platform/api.spec.yaml --output components/renku_data_services/platform/apispec.py --base-class renku_data_services.platform.apispec_base.BaseAPISpec $(codegen_params)
+components/renku_data_services/data_connectors/apispec.py: components/renku_data_services/data_connectors/api.spec.yaml
+	poetry run datamodel-codegen --input components/renku_data_services/data_connectors/api.spec.yaml --output components/renku_data_services/data_connectors/apispec.py --base-class renku_data_services.data_connectors.apispec_base.BaseAPISpec $(codegen_params)
 
 ##@ Apispec
 
-schemas: components/renku_data_services/crc/apispec.py components/renku_data_services/storage/apispec.py components/renku_data_services/users/apispec.py components/renku_data_services/project/apispec.py components/renku_data_services/namespace/apispec.py components/renku_data_services/secrets/apispec.py components/renku_data_services/connected_services/apispec.py components/renku_data_services/repositories/apispec.py components/renku_data_services/notebooks/apispec.py components/renku_data_services/platform/apispec.py  ## Generate pydantic classes from apispec yaml files
+schemas: components/renku_data_services/crc/apispec.py \
+components/renku_data_services/storage/apispec.py \
+components/renku_data_services/users/apispec.py \
+components/renku_data_services/project/apispec.py \
+components/renku_data_services/session/apispec.py \
+components/renku_data_services/namespace/apispec.py \
+components/renku_data_services/secrets/apispec.py \
+components/renku_data_services/connected_services/apispec.py \
+components/renku_data_services/repositories/apispec.py \
+components/renku_data_services/notebooks/apispec.py \
+components/renku_data_services/platform/apispec.py \
+components/renku_data_services/data_connectors/apispec.py  ## Generate pydantic classes from apispec yaml files
 	@echo "generated classes based on ApiSpec"
 
 ##@ Avro schemas
@@ -84,6 +97,8 @@ style_checks:  ## Run linting and style checks
 	@$(call test_apispec_up_to_date,"notebooks")
 	@echo "checking platform apispec is up to date"
 	@$(call test_apispec_up_to_date,"platform")
+	@echo "checking session apispec is up to date"
+	@$(call test_apispec_up_to_date,"session")
 	poetry run mypy
 	poetry run ruff format --check
 	poetry run ruff check .
@@ -131,17 +146,17 @@ help:  ## Display this help.
 
 ##@ Helm/k8s
 
-kind_cluster:  ## Creates a kind cluster for testing
-	kind delete cluster
-	docker network rm -f kind
-	docker network create -d=bridge -o com.docker.network.bridge.enable_ip_masquerade=true -o com.docker.network.driver.mtu=1500 --ipv6=false kind
-	kind create cluster --config kind_config.yaml
-	kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
-	echo "Waiting for ingress controller to initialize"
-	sleep 15
-	kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
-
-install_amaltheas:  ## Installs both version of amalthea in the currently active k8s context.
+k3d_cluster:  ## Creates a k3d cluster for testing
+	k3d cluster delete
+	k3d cluster create --agents 1 --k3s-arg --disable=metrics-server@server:0
+
+install_amaltheas:  ## Installs both version of amalthea in the. NOTE: It uses the currently active k8s context.
 	helm repo add renku https://swissdatasciencecenter.github.io/helm-charts
-	helm install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
-	helm install amalthea-sessions renku/amalthea-sessions --version $(AMALTHEA_SESSIONS_VERSION)
+	helm repo update
+	helm upgrade --install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
+	helm upgrade --install amalthea-se renku/amalthea-sessions --version ${AMALTHEA_SESSIONS_VERSION}
+
+# TODO: Add the version variables from the top of the file here when the charts are fully published
+amalthea_schema:  ## Updates generates pydantic classes from CRDs
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/feat-add-cloud-storage/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg

bases/renku_data_services/data_api/app.py

@@ -15,12 +15,14 @@
     ResourcePoolUsersBP,
     UserResourcePoolsBP,
 )
+from renku_data_services.data_connectors.blueprints import DataConnectorsBP
 from renku_data_services.namespace.blueprints import GroupsBP
+from renku_data_services.notebooks.blueprints import NotebooksBP, NotebooksNewBP
 from renku_data_services.platform.blueprints import PlatformConfigBP
 from renku_data_services.project.blueprints import ProjectsBP
 from renku_data_services.repositories.blueprints import RepositoriesBP
 from renku_data_services.session.blueprints import EnvironmentsBP, SessionLaunchersBP
-from renku_data_services.storage.blueprints import StorageBP, StorageSchemaBP, StoragesV2BP
+from renku_data_services.storage.blueprints import StorageBP, StorageSchemaBP
 from renku_data_services.users.blueprints import KCUsersBP, UserPreferencesBP, UserSecretsBP
 
 
@@ -74,12 +76,6 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         storage_repo=config.storage_repo,
         authenticator=config.gitlab_authenticator,
     )
-    storages_v2 = StoragesV2BP(
-        name="storages_v2",
-        url_prefix=url_prefix,
-        storage_v2_repo=config.storage_v2_repo,
-        authenticator=config.authenticator,
-    )
     storage_schema = StorageSchemaBP(name="storage_schema", url_prefix=url_prefix)
     user_preferences = UserPreferencesBP(
         name="user_preferences",
@@ -134,12 +130,40 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         authenticator=config.authenticator,
         internal_gitlab_authenticator=config.gitlab_authenticator,
     )
+    notebooks = NotebooksBP(
+        name="notebooks_old",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+        git_repo=config.git_repositories_repo,
+        rp_repo=config.rp_repo,
+    )
+    notebooks_new = NotebooksNewBP(
+        name="notebooks",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        project_repo=config.project_repo,
+        session_repo=config.session_repo,
+        rp_repo=config.rp_repo,
+        data_connector_repo=config.data_connector_repo,
+        data_connector_project_link_repo=config.data_connector_to_project_link_repo,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+    )
     platform_config = PlatformConfigBP(
         name="platform_config",
         url_prefix=url_prefix,
         platform_repo=config.platform_repo,
         authenticator=config.authenticator,
     )
+    data_connectors = DataConnectorsBP(
+        name="data_connectors",
+        url_prefix=url_prefix,
+        data_connector_repo=config.data_connector_repo,
+        data_connector_to_project_link_repo=config.data_connector_to_project_link_repo,
+        authenticator=config.authenticator,
+    )
     app.blueprint(
         [
             resource_pools.blueprint(),
@@ -150,7 +174,6 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
             user_secrets.blueprint(),
             user_resource_pools.blueprint(),
             storage.blueprint(),
-            storages_v2.blueprint(),
             storage_schema.blueprint(),
             user_preferences.blueprint(),
             misc.blueprint(),
@@ -161,7 +184,10 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
             oauth2_clients.blueprint(),
             oauth2_connections.blueprint(),
             repositories.blueprint(),
+            notebooks.blueprint(),
+            notebooks_new.blueprint(),
             platform_config.blueprint(),
+            data_connectors.blueprint(),
         ]
     )