Merge branch 'main' into leafty/add-disk-size-session-envs

bases/renku_data_services/data_api/app.py

@@ -159,6 +159,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         session_repo=config.session_repo,
         storage_repo=config.storage_repo,
         rp_repo=config.rp_repo,
+        user_repo=config.kc_user_repo,
         data_connector_repo=config.data_connector_repo,
         data_connector_project_link_repo=config.data_connector_to_project_link_repo,
         data_connector_secret_repo=config.data_connector_secret_repo,

bases/renku_data_services/data_api/main.py

@@ -7,8 +7,9 @@
 
 import sentry_sdk
 import uvloop
-from sanic import Sanic
+from sanic import Request, Sanic
 from sanic.log import logger
+from sanic.response import BaseHTTPResponse
 from sanic.worker.loader import AppLoader
 from sentry_sdk.integrations.asyncio import AsyncioIntegration
 from sentry_sdk.integrations.grpc import GRPCIntegration
@@ -124,6 +125,12 @@ async def setup_sentry(_: Sanic) -> None:
 
     app.register_middleware(validate_null_byte, "request")
 
+    @app.middleware("response")
+    async def handle_head(request: Request, response: BaseHTTPResponse) -> None:
+        """Make sure HEAD requests return an empty body."""
+        if request.method == "HEAD":
+            response.body = None
+
     @app.main_process_start
     async def do_migrations(_: Sanic) -> None:
         logger.info("running migrations")

components/renku_data_services/app_config/config.py

@@ -199,68 +199,30 @@ def load_apispec() -> dict[str, Any]:
         Note: loading these files takes quite some time and is repeated for each test. Having
         them cached in this method reduces that time significantly.
         """
+        files = [
+            renku_data_services.crc.__file__,
+            renku_data_services.storage.__file__,
+            renku_data_services.users.__file__,
+            renku_data_services.project.__file__,
+            renku_data_services.namespace.__file__,
+            renku_data_services.session.__file__,
+            renku_data_services.connected_services.__file__,
+            renku_data_services.repositories.__file__,
+            renku_data_services.notebooks.__file__,
+            renku_data_services.platform.__file__,
+            renku_data_services.message_queue.__file__,
+            renku_data_services.data_connectors.__file__,
+        ]
+
+        api_specs = []
+
         # NOTE: Read spec files required for Swagger
-        spec_file = Path(renku_data_services.crc.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            crc_spec = safe_load(f)
-
-        spec_file = Path(renku_data_services.storage.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            storage_spec = safe_load(f)
-
-        spec_file = Path(renku_data_services.users.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            users = safe_load(f)
-
-        spec_file = Path(renku_data_services.project.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            projects = safe_load(f)
-
-        spec_file = Path(renku_data_services.namespace.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            groups = safe_load(f)
-
-        spec_file = Path(renku_data_services.session.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            sessions = safe_load(f)
-
-        spec_file = Path(renku_data_services.connected_services.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            connected_services = safe_load(f)
-
-        spec_file = Path(renku_data_services.repositories.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            repositories = safe_load(f)
-
-        spec_file = Path(renku_data_services.notebooks.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            repositories = safe_load(f)
-
-        spec_file = Path(renku_data_services.platform.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            platform = safe_load(f)
-
-        spec_file = Path(renku_data_services.message_queue.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            search = safe_load(f)
-
-        spec_file = Path(renku_data_services.data_connectors.__file__).resolve().parent / "api.spec.yaml"
-        with open(spec_file) as f:
-            data_connectors = safe_load(f)
-
-        return merge_api_specs(
-            crc_spec,
-            storage_spec,
-            users,
-            projects,
-            groups,
-            sessions,
-            connected_services,
-            repositories,
-            platform,
-            search,
-            data_connectors,
-        )
+        for file in files:
+            spec_file = Path(file).resolve().parent / "api.spec.yaml"
+            with open(spec_file) as f:
+                api_specs.append(safe_load(f))
+
+        return merge_api_specs(*api_specs)
 
     def __post_init__(self) -> None:
         self.spec = self.load_apispec()

components/renku_data_services/authz/authz.py

@@ -118,18 +118,18 @@ def extend(self, other: "_AuthzChange") -> None:
 class _Relation(StrEnum):
     """Relations for Authzed."""
 
-    owner: str = "owner"
-    editor: str = "editor"
-    viewer: str = "viewer"
-    public_viewer: str = "public_viewer"
-    admin: str = "admin"
-    project_platform: str = "project_platform"
-    group_platform: str = "group_platform"
-    user_namespace_platform: str = "user_namespace_platform"
-    project_namespace: str = "project_namespace"
-    data_connector_platform: str = "data_connector_platform"
-    data_connector_namespace: str = "data_connector_namespace"
-    linked_to: str = "linked_to"
+    owner = "owner"
+    editor = "editor"
+    viewer = "viewer"
+    public_viewer = "public_viewer"
+    admin = "admin"
+    project_platform = "project_platform"
+    group_platform = "group_platform"
+    user_namespace_platform = "user_namespace_platform"
+    project_namespace = "project_namespace"
+    data_connector_platform = "data_connector_platform"
+    data_connector_namespace = "data_connector_namespace"
+    linked_to = "linked_to"
 
     @classmethod
     def from_role(cls, role: Role) -> "_Relation":
@@ -156,25 +156,25 @@ def to_role(self) -> Role:
 class ResourceType(StrEnum):
     """All possible resources stored in Authzed."""
 
-    project: str = "project"
-    user: str = "user"
-    anonymous_user: str = "anonymous_user"
-    platform: str = "platform"
-    group: str = "group"
-    user_namespace: str = "user_namespace"
-    data_connector: str = "data_connector"
+    project = "project"
+    user = "user"
+    anonymous_user = "anonymous_user"
+    platform = "platform"
+    group = "group"
+    user_namespace = "user_namespace"
+    data_connector = "data_connector"
 
 
 class AuthzOperation(StrEnum):
     """The type of change that requires authorization database update."""
 
-    create: str = "create"
-    delete: str = "delete"
-    update: str = "update"
-    update_or_insert: str = "update_or_insert"
-    insert_many: str = "insert_many"
-    create_link: str = "create_link"
-    delete_link: str = "delete_link"
+    create = "create"
+    delete = "delete"
+    update = "update"
+    update_or_insert = "update_or_insert"
+    insert_many = "insert_many"
+    create_link = "create_link"
+    delete_link = "delete_link"
 
 
 class _AuthzConverter:

components/renku_data_services/authz/models.py

@@ -16,9 +16,9 @@
 class Role(Enum):
     """Membership role."""
 
-    OWNER: str = "owner"
-    VIEWER: str = "viewer"
-    EDITOR: str = "editor"
+    OWNER = "owner"
+    VIEWER = "viewer"
+    EDITOR = "editor"
 
     @classmethod
     def from_group_role(cls, role: GroupRole) -> "Role":
@@ -36,11 +36,11 @@ def from_group_role(cls, role: GroupRole) -> "Role":
     def to_group_role(self) -> GroupRole:
         """Convert a group role into an authorization role."""
         match self:
-            case self.OWNER:
+            case Role.OWNER:
                 return GroupRole.owner
-            case self.EDITOR:
+            case Role.EDITOR:
                 return GroupRole.editor
-            case self.VIEWER:
+            case Role.VIEWER:
                 return GroupRole.viewer
             case _:
                 raise errors.ProgrammingError(message=f"Could not convert role {self} into a group role")
@@ -49,13 +49,13 @@ def to_group_role(self) -> GroupRole:
 class Scope(Enum):
     """Types of permissions - i.e. scope."""
 
-    READ: str = "read"
-    WRITE: str = "write"
-    DELETE: str = "delete"
-    CHANGE_MEMBERSHIP: str = "change_membership"
-    READ_CHILDREN: str = "read_children"
-    ADD_LINK: str = "add_link"
-    IS_ADMIN: str = "is_admin"
+    READ = "read"
+    WRITE = "write"
+    DELETE = "delete"
+    CHANGE_MEMBERSHIP = "change_membership"
+    READ_CHILDREN = "read_children"
+    ADD_LINK = "add_link"
+    IS_ADMIN = "is_admin"
 
 
 @dataclass
@@ -80,9 +80,9 @@ class Member(UnsavedMember):
 class Change(Enum):
     """The type of change executed on a specific resource."""
 
-    UPDATE: str = "update"
-    ADD: str = "add"
-    REMOVE: str = "remove"
+    UPDATE = "update"
+    ADD = "add"
+    REMOVE = "remove"
 
 
 @dataclass
@@ -96,8 +96,8 @@ class MembershipChange:
 class Visibility(Enum):
     """The visibility of a resource."""
 
-    PUBLIC: str = "public"
-    PRIVATE: str = "private"
+    PUBLIC = "public"
+    PRIVATE = "private"
 
 
 @dataclass

components/renku_data_services/message_queue/redis_queue.py

@@ -61,7 +61,7 @@ async def message_wrapper(self: _WithMessageQueue, *args: _P.args, **kwargs: _P.
                 )
             result = await f(self, *args, **kwargs)
             if result is None:
-                return result  # type: ignore[unreachable]
+                return result
             events = EventConverter.to_events(result, event_type)
 
             for event in events:

components/renku_data_services/namespace/models.py

@@ -58,8 +58,8 @@ class GroupMemberDetails:
 class NamespaceKind(str, Enum):
     """Allowed kinds of namespaces."""
 
-    group: str = "group"
-    user: str = "user"
+    group = "group"
+    user = "user"
 
 
 @dataclass

components/renku_data_services/notebooks/api/classes/image.py

@@ -16,10 +16,10 @@
 class ManifestTypes(Enum):
     """The mime types for docker image manifests."""
 
-    docker_v2: str = "application/vnd.docker.distribution.manifest.v2+json"
-    docker_v2_list: str = "application/vnd.docker.distribution.manifest.list.v2+json"
-    oci_v1_manifest: str = "application/vnd.oci.image.manifest.v1+json"
-    oci_v1_index: str = "application/vnd.oci.image.index.v1+json"
+    docker_v2 = "application/vnd.docker.distribution.manifest.v2+json"
+    docker_v2_list = "application/vnd.docker.distribution.manifest.list.v2+json"
+    oci_v1_manifest = "application/vnd.oci.image.manifest.v1+json"
+    oci_v1_index = "application/vnd.oci.image.index.v1+json"
 
 
 DEFAULT_PLATFORM_ARCHITECTURE = "amd64"

components/renku_data_services/notebooks/api/schemas/servers_get.py

@@ -33,10 +33,10 @@ def list(cls) -> list[str]:
 class StepStatusEnum(Enum):
     """Enum for status of a session start step."""
 
-    ready: str = "ready"  # An init job completely done or container fully running
-    waiting: str = "waiting"  # Waiting to start
-    executing: str = "executing"  # Running but not complete or fully ready
-    failed: str = "failed"
+    ready = "ready"  # An init job completely done or container fully running
+    waiting = "waiting"  # Waiting to start
+    executing = "executing"  # Running but not complete or fully ready
+    failed = "failed"
 
     @classmethod
     def list(cls) -> list[str]:

components/renku_data_services/notebooks/blueprints.py

@@ -68,7 +68,6 @@
     renku_2_make_server_name,
 )
 from renku_data_services.notebooks.utils import (
-    get_user_secret,
     merge_node_affinities,
     node_affinity_from_resource_class,
     tolerations_from_resource_class,
@@ -78,6 +77,7 @@
 from renku_data_services.session.db import SessionRepository
 from renku_data_services.storage.db import StorageRepository
 from renku_data_services.users.db import UserRepo
+from renku_data_services.utils.cryptography import get_encryption_key
 
 
 @dataclass(kw_only=True)
@@ -241,6 +241,7 @@ class NotebooksNewBP(CustomBlueprint):
     session_repo: SessionRepository
     rp_repo: ResourcePoolRepository
     storage_repo: StorageRepository
+    user_repo: UserRepo
     data_connector_repo: DataConnectorRepository
     data_connector_project_link_repo: DataConnectorProjectLinkRepository
     data_connector_secret_repo: DataConnectorSecretRepository
@@ -340,7 +341,9 @@ async def _handler(
             data_sources: list[DataSource] = []
             user_secret_key: str | None = None
             if isinstance(user, AuthenticatedAPIUser) and len(dcs_secrets) > 0:
-                user_secret_key = await get_user_secret(self.nb_config.data_service_url, user)
+                secret_key = await self.user_repo.get_or_create_user_secret_key(requested_by=user)
+                user_secret_key = get_encryption_key(secret_key.encode(), user.id.encode()).decode("utf-8")
+
             for cs_id, cs in dcs.items():
                 secret_name = f"{server_name}-ds-{cs_id.lower()}"
                 secret_key_needed = len(dcs_secrets.get(cs_id, [])) > 0

components/renku_data_services/notebooks/config/dynamic.py

@@ -37,9 +37,9 @@ def _parse_value_as_float(val: Any) -> float:
 class CPUEnforcement(str, Enum):
     """CPU enforcement policies."""
 
-    LAX: str = "lax"  # CPU limit equals 3x cpu request
-    STRICT: str = "strict"  # CPU limit equals cpu request
-    OFF: str = "off"  # no CPU limit at all
+    LAX = "lax"  # CPU limit equals 3x cpu request
+    STRICT = "strict"  # CPU limit equals cpu request
+    OFF = "off"  # no CPU limit at all
 
 
 @dataclass

components/renku_data_services/notebooks/util/kubernetes_.py

@@ -100,8 +100,8 @@ def _make_server_name_prefix(safe_username: str) -> str:
 class PatchKind(StrEnum):
     """Content types for different json patches."""
 
-    json: str = "application/json-patch+json"
-    merge: str = "application/merge-patch+json"
+    json = "application/json-patch+json"
+    merge = "application/merge-patch+json"
 
 
 def find_container(patches: list[Patch], container_name: str) -> dict[str, Any] | None: