Skip to content

Commit

Permalink
squashme: notebooks changes
Browse files Browse the repository at this point in the history
This includes major edits to the notebooks code to work with the data
service.
  • Loading branch information
olevski committed Aug 27, 2024
1 parent 1e99745 commit 1fe17ee
Show file tree
Hide file tree
Showing 68 changed files with 6,413 additions and 1,395 deletions.
5 changes: 4 additions & 1 deletion .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
"minikube": "none"
},
"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {},
"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
"jqVersion": "latest",
"yqVersion": "latest"
},
"ghcr.io/dhoeric/features/k9s:1": {},
"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
"command": "alias k=kubectl"
Expand Down
3 changes: 3 additions & 0 deletions .devcontainer/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ services:
ZED_TOKEN: renku
ZED_INSECURE: "true"
POETRY_CACHE_DIR: "/poetry_cache"
NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
network_mode: service:db
depends_on:
- db
Expand All @@ -43,6 +45,7 @@ services:
- "8080:8080"
- "5678:5678"
- "50051:50051"
- "8888:80"

swagger:
image: swaggerapi/swagger-ui
Expand Down
14 changes: 9 additions & 5 deletions .github/workflows/acceptance-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,15 @@ jobs:
renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
persist: ${{ steps.deploy-comment.outputs.persist}}
extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
steps:
- id: deploy-comment
uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3
uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1
with:
string: /deploy
pr_ref: ${{ github.event.number }}
Expand Down Expand Up @@ -61,7 +63,7 @@ jobs:
body: |
You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
- name: Build and deploy
uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3
uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1
env:
RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
Expand All @@ -84,14 +86,16 @@ jobs:
renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
renku_data_services: "@${{ github.head_ref }}"
amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
amalthea-sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
extra_values: "${{ needs.check-deploy.outputs.extra-values }}"

selenium-acceptance-tests:
needs: [check-deploy, deploy-pr]
if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
runs-on: ubuntu-22.04
steps:
- uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3
- uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1
with:
kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
renku-release: renku-ci-ds-${{ github.event.number }}
Expand All @@ -118,7 +122,7 @@ jobs:
steps:
- name: Extract Renku repository reference
run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
- uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3
- uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1
with:
e2e-target: ${{ matrix.tests }}
renku-reference: ${{ env.RENKU_REFERENCE }}
Expand Down Expand Up @@ -146,7 +150,7 @@ jobs:
body: |
Tearing down the temporary RenkuLab deplyoment for this PR.
- name: renku teardown
uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3
uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1
env:
HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}
Expand Down
41 changes: 41 additions & 0 deletions .github/workflows/save_cache.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Create cache from commits on main

on:
push:
branches:
- main
- chore-add-kind
workflow_dispatch:


jobs:
save-poetry-cache:
runs-on: ubuntu-latest
env:
CACHE_KEY: main-branch-poetry-cache-ubuntu
CACHE_PATH: .devcontainer/.poetry_cache
DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install python deps
uses: devcontainers/[email protected]
with:
runCmd: poetry install --with dev
push: always
skipContainerUserIdUpdate: false
imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
- uses: actions/cache/save@v3
name: Create cache
with:
path: ${{ env.CACHE_PATH }}
key: ${{ env.CACHE_KEY }}
62 changes: 62 additions & 0 deletions .github/workflows/test_publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,58 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Docker image metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.DEVCONTAINER_REGISTRY }}/${{ env.DEVCONTAINER_IMAGE_NAME }}
tags: |
type=ref,event=pr,prefix=cache-pr-,priority=600
type=ref,event=branch,prefix=cache-,priority=500
type=ref,event=tag,prefix=cache-,priority=500
flavor: |
latest=false
- name: Extract Docker image name
id: docker_image
env:
IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
run: |
IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1)
IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1)
IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2)
echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT"
echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Pre-build devcontainer
uses: devcontainers/[email protected]
continue-on-error: true
with:
push: always
skipContainerUserIdUpdate: false
imageName: ${{ steps.docker_image.outputs.image_repository }}
imageTag: ${{ steps.docker_image.outputs.image_tag }}
cacheFrom: |
${{ steps.docker_image.outputs.image_repository }}:${{ steps.docker_image.outputs.image_tag }}
${{ steps.docker_image.outputs.image_repository }}:${{ env.DEVCONTAINER_IMAGE_TAG_MAIN }}
style-checks:
runs-on: ubuntu-latest
needs:
- build-devcontainer
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/cache/restore@v3
name: Restore cache
with:
path: ${{ env.CACHE_PATH }}
key: ${{ env.CACHE_KEY }}
- name: Set Git config
shell: bash
run: |
Expand Down Expand Up @@ -99,6 +151,11 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/cache/restore@v3
name: Restore cache
with:
path: ${{ env.CACHE_PATH }}
key: ${{ env.CACHE_KEY }}
- name: Set Git config
shell: bash
run: |
Expand Down Expand Up @@ -143,6 +200,11 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/cache/restore@v3
name: Restore cache
with:
path: ${{ env.CACHE_PATH }}
key: ${{ env.CACHE_KEY }}
- name: Set Git config
shell: bash
run: |
Expand Down
16 changes: 11 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro kind_cluster install_amaltheas all

AMALTHEA_JS_VERSION ?= 0.11.0
AMALTHEA_SESSIONS_VERSION ?= 0.0.1-new-operator-chart
AMALTHEA_JS_VERSION ?= 0.12.2
AMALTHEA_SESSIONS_VERSION ?= 0.0.9-new-operator-chart
codegen_params = --input-file-type openapi --output-model-type pydantic_v2.BaseModel --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --set-default-enum-member --openapi-scopes schemas paths parameters --set-default-enum-member --use-one-literal-as-default --use-default

define test_apispec_up_to_date
Expand Down Expand Up @@ -153,7 +153,13 @@ kind_cluster: ## Creates a kind cluster for testing
sleep 15
kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s

install_amaltheas: ## Installs both version of amalthea in the currently active k8s context.
install_amaltheas: ## Installs both version of amalthea in the. NOTE: It uses the currently active k8s context.
helm repo add renku https://swissdatasciencecenter.github.io/helm-charts
helm install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
helm install amalthea-sessions renku/amalthea-sessions --version $(AMALTHEA_SESSIONS_VERSION)
helm repo update
helm upgrade --install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
helm upgrade --install amalthea-sessions amalthea-sessions-0.0.9-new-operator-chart.tgz --version $(AMALTHEA_SESSIONS_VERSION)

# TODO: Add the version variables from the top of the file here when the charts are fully published
amalthea_schema: ## Updates generates pydantic classes from CRDs
curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/fix-missing-status/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
21 changes: 21 additions & 0 deletions bases/renku_data_services/data_api/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
UserResourcePoolsBP,
)
from renku_data_services.namespace.blueprints import GroupsBP
from renku_data_services.notebooks.blueprints import NotebooksBP, NotebooksNewBP
from renku_data_services.platform.blueprints import PlatformConfigBP
from renku_data_services.project.blueprints import ProjectsBP
from renku_data_services.repositories.blueprints import RepositoriesBP
Expand Down Expand Up @@ -134,6 +135,24 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
authenticator=config.authenticator,
internal_gitlab_authenticator=config.gitlab_authenticator,
)
notebooks = NotebooksBP(
name="notebooks_old",
url_prefix=url_prefix,
authenticator=config.authenticator,
nb_config=config.nb_config,
internal_gitlab_authenticator=config.gitlab_authenticator,
git_repo=config.git_repositories_repo,
)
notebooks_new = NotebooksNewBP(
name="notebooks",
url_prefix=url_prefix,
authenticator=config.authenticator,
nb_config=config.nb_config,
project_repo=config.project_repo,
session_repo=config.session_repo,
rp_repo=config.rp_repo,
internal_gitlab_authenticator=config.gitlab_authenticator,
)
platform_config = PlatformConfigBP(
name="platform_config",
url_prefix=url_prefix,
Expand Down Expand Up @@ -161,6 +180,8 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
oauth2_clients.blueprint(),
oauth2_connections.blueprint(),
repositories.blueprint(),
notebooks.blueprint(),
notebooks_new.blueprint(),
platform_config.blueprint(),
]
)
Expand Down
12 changes: 10 additions & 2 deletions components/renku_data_services/app_config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from renku_data_services.message_queue.interface import IMessageQueue
from renku_data_services.message_queue.redis_queue import RedisQueue
from renku_data_services.namespace.db import GroupRepository
from renku_data_services.notebooks.config import _NotebooksConfig
from renku_data_services.platform.db import PlatformRepository
from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository
from renku_data_services.repositories.db import GitRepositoriesRepository
Expand Down Expand Up @@ -144,6 +145,7 @@ class Config:
kc_api: IKeycloakAPI
message_queue: IMessageQueue
gitlab_url: str | None
nb_config: _NotebooksConfig

secrets_service_public_key: rsa.RSAPublicKey
"""The public key of the secrets service, used to encrypt user secrets that only it can decrypt."""
Expand Down Expand Up @@ -208,6 +210,10 @@ def __post_init__(self) -> None:
with open(spec_file) as f:
repositories = safe_load(f)

spec_file = Path(renku_data_services.notebooks.__file__).resolve().parent / "api.spec.yaml"
with open(spec_file) as f:
repositories = safe_load(f)

spec_file = Path(renku_data_services.platform.__file__).resolve().parent / "api.spec.yaml"
with open(spec_file) as f:
platform = safe_load(f)
Expand Down Expand Up @@ -408,8 +414,8 @@ def from_env(cls, prefix: str = "") -> "Config":
gitlab_client: base_models.GitlabAPIProtocol
user_preferences_config: UserPreferencesConfig
version = os.environ.get(f"{prefix}VERSION", "0.0.1")
server_options_file = os.environ.get("SERVER_OPTIONS")
server_defaults_file = os.environ.get("SERVER_DEFAULTS")
server_options_file = os.environ.get("NB_SERVER_OPTIONS__UI_CHOICES_PATH")
server_defaults_file = os.environ.get("NB_SERVER_OPTIONS__DEFAULTS_PATH")
k8s_namespace = os.environ.get("K8S_NAMESPACE", "default")
max_pinned_projects = int(os.environ.get(f"{prefix}MAX_PINNED_PROJECTS", "10"))
user_preferences_config = UserPreferencesConfig(max_pinned_projects=max_pinned_projects)
Expand Down Expand Up @@ -491,6 +497,7 @@ def from_env(cls, prefix: str = "") -> "Config":
sentry = SentryConfig.from_env(prefix)
trusted_proxies = TrustedProxiesConfig.from_env(prefix)
message_queue = RedisQueue(redis)
nb_config = _NotebooksConfig.from_env(db)

return cls(
version=version,
Expand All @@ -511,4 +518,5 @@ def from_env(cls, prefix: str = "") -> "Config":
encryption_key=encryption_key,
secrets_service_public_key=secrets_service_public_key,
gitlab_url=gitlab_url,
nb_config=nb_config,
)
3 changes: 2 additions & 1 deletion components/renku_data_services/authn/dummy.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,12 @@ async def authenticate(access_token: str, request: Request) -> base_models.APIUs
)

return base_models.APIUser(
is_admin=user_props.get("is_admin", False),
is_admin_init=user_props.get("is_admin", False),
id=user_props.get("id", "some-id") if is_set else None,
access_token=access_token,
first_name=user_props.get("first_name", "John") if is_set else None,
last_name=user_props.get("last_name", "Doe") if is_set else None,
email=user_props.get("email", "[email protected]") if is_set else None,
full_name=user_props.get("full_name", "John Doe") if is_set else None,
refresh_token=request.headers.get("Renku-Auth-Refresh-Token"),
)
Loading

0 comments on commit 1fe17ee

Please sign in to comment.