Merge pull request #92 from neptune-ai/kg/create-project-api

Add `neptune_scale.projects.create_project()`

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Neptune will now skip non-finite metric values by default, instead of raising an error. This can be configured using
   the `NEPTUNE_SKIP_NON_FINITE_METRICS` environment variable ([#85](https://github.com/neptune-ai/neptune-client-scale/pull/85))
+- New function: `neptune_scale.projects.create_project()` ([#92](https://github.com/neptune-ai/neptune-client-scale/pull/92))
 
 ## [0.7.2] - 2024-11-08
 

src/neptune_scale/exceptions.py

@@ -51,7 +51,8 @@ class NeptuneScaleError(Exception):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         ensure_style_detected()
-        super().__init__(self.message.format(*args, **STYLES, **kwargs))
+        message = kwargs.pop("message", self.message)
+        super().__init__(message.format(*args, **STYLES, **kwargs))
 
 
 class NeptuneScaleWarning(Warning):
@@ -62,6 +63,25 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(self.message.format(*args, **STYLES, **kwargs))
 
 
+class NeptuneBadRequestError(NeptuneScaleError):
+    """
+    A generic "bad request" error. Pass `reason` to provide a custom message.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        reason = kwargs.get("reason", None)
+        if not reason:
+            reason = "The request contains invalid data"
+        kwargs["reason"] = reason
+        kwargs["status_code"] = kwargs.get("status_code", 400)
+        super().__init__(*args, **kwargs)
+
+    message = """
+{h1}
+NeptuneBadRequestError({status_code}): {reason}
+"""
+
+
 class NeptuneSynchronizationStopped(NeptuneScaleError):
     message = "Internal synchronization process was stopped."
 
@@ -206,6 +226,14 @@ class NeptuneProjectInvalidName(NeptuneScaleError):
 """
 
 
+class NeptuneProjectAlreadyExists(NeptuneScaleError):
+    message = """
+{h1}
+NeptuneProjectAlreadyExists: A project with the provided name or project key already exists.
+{end}
+"""
+
+
 class NeptuneRunNotFound(NeptuneScaleError):
     message = """
 {h1}

src/neptune_scale/net/api_client.py

@@ -15,7 +15,7 @@
 #
 from __future__ import annotations
 
-__all__ = ("HostedApiClient", "MockedApiClient", "ApiClient", "backend_factory")
+__all__ = ("HostedApiClient", "MockedApiClient", "ApiClient", "backend_factory", "with_api_errors_handling")
 
 import abc
 import os
@@ -24,9 +24,11 @@
 from http import HTTPStatus
 from typing import (
     Any,
+    Callable,
     Literal,
 )
 
+import httpx
 from httpx import Timeout
 from neptune_api import (
     AuthenticatedClient,
@@ -39,6 +41,13 @@
 )
 from neptune_api.auth_helpers import exchange_api_key
 from neptune_api.credentials import Credentials
+from neptune_api.errors import (
+    InvalidApiTokenException,
+    UnableToDeserializeApiKeyError,
+    UnableToExchangeApiKeyError,
+    UnableToRefreshTokenError,
+    UnexpectedStatus,
+)
 from neptune_api.models import (
     ClientConfig,
     Error,
@@ -55,6 +64,11 @@
 from neptune_api.proto.neptune_pb.ingest.v1.pub.request_status_pb2 import RequestStatus
 from neptune_api.types import Response
 
+from neptune_scale.exceptions import (
+    NeptuneConnectionLostError,
+    NeptuneInvalidCredentialsError,
+    NeptuneUnableToAuthenticateError,
+)
 from neptune_scale.sync.parameters import REQUEST_TIMEOUT
 from neptune_scale.util.abstract import Resource
 from neptune_scale.util.envs import ALLOW_SELF_SIGNED_CERTIFICATE
@@ -123,24 +137,24 @@ def __init__(self, api_token: str) -> None:
 
         logger.debug("Trying to connect to Neptune API")
         config, token_urls = get_config_and_token_urls(credentials=credentials, verify_ssl=verify_ssl)
-        self._backend = create_auth_api_client(
+        self.backend = create_auth_api_client(
             credentials=credentials, config=config, token_refreshing_urls=token_urls, verify_ssl=verify_ssl
         )
         logger.debug("Connected to Neptune API")
 
     def submit(self, operation: RunOperation, family: str) -> Response[SubmitResponse]:
-        return submit_operation.sync_detailed(client=self._backend, body=operation, family=family)
+        return submit_operation.sync_detailed(client=self.backend, body=operation, family=family)
 
     def check_batch(self, request_ids: list[str], project: str) -> Response[BulkRequestStatus]:
         return check_request_status_bulk.sync_detailed(
-            client=self._backend,
+            client=self.backend,
             project_identifier=project,
             body=RequestIdList(ids=[RequestId(value=request_id) for request_id in request_ids]),
         )
 
     def close(self) -> None:
         logger.debug("Closing API client")
-        self._backend.__exit__()
+        self.backend.__exit__()
 
 
 class MockedApiClient(ApiClient):
@@ -171,3 +185,19 @@ def backend_factory(api_token: str, mode: Literal["async", "disabled"]) -> ApiCl
     if mode == "disabled":
         return MockedApiClient()
     return HostedApiClient(api_token=api_token)
+
+
+def with_api_errors_handling(func: Callable[..., Any]) -> Callable[..., Any]:
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except (InvalidApiTokenException, UnableToDeserializeApiKeyError):
+            raise NeptuneInvalidCredentialsError()
+        except (UnableToRefreshTokenError, UnableToExchangeApiKeyError, UnexpectedStatus):
+            raise NeptuneUnableToAuthenticateError()
+        except (httpx.ConnectError, httpx.TimeoutException, httpx.RemoteProtocolError):
+            raise NeptuneConnectionLostError()
+        except Exception as e:
+            raise e
+
+    return wrapper

src/neptune_scale/net/projects.py

@@ -0,0 +1,84 @@
+import os
+import re
+from enum import Enum
+from json import JSONDecodeError
+from typing import (
+    Any,
+    Optional,
+)
+
+import httpx
+
+from neptune_scale.exceptions import (
+    NeptuneApiTokenNotProvided,
+    NeptuneBadRequestError,
+    NeptuneProjectAlreadyExists,
+)
+from neptune_scale.net.api_client import (
+    HostedApiClient,
+    with_api_errors_handling,
+)
+from neptune_scale.util.envs import API_TOKEN_ENV_NAME
+
+PROJECTS_PATH_BASE = "/api/backend/v1/projects"
+
+
+class ProjectVisibility(Enum):
+    PRIVATE = "priv"
+    PUBLIC = "pub"
+    WORKSPACE = "workspace"
+
+
+ORGANIZATION_NOT_FOUND_RE = re.compile(r"Organization .* not found")
+
+
+@with_api_errors_handling
+def create_project(
+    workspace: str,
+    name: str,
+    *,
+    visibility: ProjectVisibility = ProjectVisibility.PRIVATE,
+    description: Optional[str] = None,
+    key: Optional[str] = None,
+    fail_if_exists: bool = False,
+    api_token: Optional[str] = None,
+) -> None:
+    api_token = api_token or os.environ.get(API_TOKEN_ENV_NAME)
+    if api_token is None:
+        raise NeptuneApiTokenNotProvided()
+
+    client = HostedApiClient(api_token=api_token)
+    visibility = ProjectVisibility(visibility)
+
+    body = {
+        "name": name,
+        "description": description,
+        "projectKey": key,
+        "organizationIdentifier": workspace,
+        "visibility": visibility.value,
+    }
+
+    response = client.backend.get_httpx_client().request("post", PROJECTS_PATH_BASE, json=body)
+    json = _safe_json(response)
+
+    try:
+        response.raise_for_status()
+    except httpx.HTTPStatusError as e:
+        code = e.response.status_code
+        if code == 409:
+            if fail_if_exists:
+                raise NeptuneProjectAlreadyExists()
+        # We need to match plain text, as this is what the backend returns
+        elif code == 404 and ORGANIZATION_NOT_FOUND_RE.match(response.text):
+            raise NeptuneBadRequestError(status_code=code, reason=f"Workspace '{workspace}' not found")
+        elif code // 100 == 4:
+            raise NeptuneBadRequestError(status_code=code, reason=json.get("message"))
+        else:
+            raise e
+
+
+def _safe_json(response: httpx.Response) -> Any:
+    try:
+        return response.json()
+    except JSONDecodeError:
+        return {}

src/neptune_scale/projects.py

@@ -0,0 +1,117 @@
+import re
+from typing import (
+    Optional,
+    Tuple,
+    cast,
+)
+
+from neptune_scale.api.validation import verify_type
+from neptune_scale.net import projects
+from neptune_scale.net.projects import ProjectVisibility
+
+PROJECT_QUALIFIED_NAME_RE = re.compile(r"^((?P<workspace>[\w\-.]+)/)?(?P<project>[\w\-.]+)$")
+
+
+def create_project(
+    name: str,
+    *,
+    workspace: Optional[str] = None,
+    visibility: str = ProjectVisibility.PRIVATE.value,
+    description: Optional[str] = None,
+    key: Optional[str] = None,
+    fail_if_exists: bool = False,
+    api_token: Optional[str] = None,
+) -> str:
+    """Creates a new project in a Neptune workspace.
+
+    Args:
+        name (str): Name of the project. Can contain letters and hyphens (-). For example, "project-x".
+        workspace (str, optional): Name of your Neptune workspace.
+            You can omit this argument if you include the workspace name in the `name` argument.
+        visibility: Level of privacy for the project. Options:
+            - "pub": Public. Anyone on the internet can see it.
+            - "priv" (default): Private. Only users specifically assigned to the project can access it. Requires a plan with
+                project-level access control.
+            - "workspace" (team workspaces only): Accessible to all workspace members.
+        description: Project description. If None, it's left empty.
+        key: Project identifier. Must contain 1-10 upper case letters or numbers (at least one letter).
+            For example, "PX2". If you leave it out, Neptune generates a project key for you.
+        fail_if_exists: If the project already exists and this flag is set to `True`, an error is raised.
+        api_token: Account's API token.
+            If not provided, the value of the NEPTUNE_API_TOKEN environment variable is used (recommended).
+
+    Returns:
+        The name of the new project created.
+    """
+
+    verify_type("name", name, str)
+    verify_type("workspace", workspace, (str, type(None)))
+    verify_type("visibility", visibility, str)
+    verify_type("description", description, (str, type(None)))
+    verify_type("key", key, (str, type(None)))
+    verify_type("api_token", api_token, (str, type(None)))
+
+    workspace, name = extract_workspace_and_project(name=name, workspace=workspace)
+    projects.create_project(
+        workspace=workspace,
+        name=name,
+        visibility=visibility,
+        description=description,
+        key=key,
+        fail_if_exists=fail_if_exists,
+        api_token=api_token,
+    )
+
+    return normalize_project_name(name, workspace)
+
+
+def extract_workspace_and_project(name: str, workspace: Optional[str] = None) -> Tuple[str, str]:
+    """Return a tuple of (workspace name, project name) from the provided
+    fully qualified project name, or a name + workspace
+
+    >>> extract_workspace_and_project("my-own.workspace_/pr_oj-ect")
+    ('my-own.workspace_', 'pr_oj-ect')
+    >>> extract_workspace_and_project("project", "workspace")
+    ('workspace', 'project')
+    >>> extract_workspace_and_project("workspace/project", "workspace")
+    ('workspace', 'project')
+    >>> extract_workspace_and_project("workspace/project", "another_workspace")
+    Traceback (most recent call last):
+    ...
+    ValueError: The provided `workspace` argument `another_workspace` is different ...
+    >>> extract_workspace_and_project("project")
+    Traceback (most recent call last):
+    ...
+    ValueError: Workspace not provided ...
+    >>> extract_workspace_and_project("workspace/project!@#")
+    Traceback (most recent call last):
+    ...
+    ValueError: Invalid project name ...
+    """
+    project_spec = PROJECT_QUALIFIED_NAME_RE.search(name)
+
+    if not project_spec:
+        raise ValueError(f"Invalid project name `{name}`")
+
+    extracted_workspace, extracted_project_name = (
+        project_spec["workspace"],
+        project_spec["project"],
+    )
+
+    if not workspace and not extracted_workspace:
+        raise ValueError("Workspace not provided in neither project name or the `workspace` parameter.")
+
+    if workspace and extracted_workspace and workspace != extracted_workspace:
+        raise ValueError(
+            f"The provided `workspace` argument `{workspace}` is different from the one in project name `{name}`"
+        )
+
+    final_workspace_name = cast(str, extracted_workspace or workspace)
+
+    return final_workspace_name, extracted_project_name
+
+
+def normalize_project_name(name: str, workspace: Optional[str] = None) -> str:
+    extracted_workspace_name, extracted_project_name = extract_workspace_and_project(name=name, workspace=workspace)
+
+    return f"{extracted_workspace_name}/{extracted_project_name}"