diff --git a/CHANGELOG.md b/CHANGELOG.md index 0319b5aa..da605173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Neptune will now skip non-finite metric values by default, instead of raising an error. This can be configured using the `NEPTUNE_SKIP_NON_FINITE_METRICS` environment variable ([#85](https://github.com/neptune-ai/neptune-client-scale/pull/85)) +- New function: `neptune_scale.projects.create_project()` ([#92](https://github.com/neptune-ai/neptune-client-scale/pull/92)) ## [0.7.2] - 2024-11-08 diff --git a/src/neptune_scale/exceptions.py b/src/neptune_scale/exceptions.py index 5c629404..f1e6aa7e 100644 --- a/src/neptune_scale/exceptions.py +++ b/src/neptune_scale/exceptions.py @@ -51,7 +51,8 @@ class NeptuneScaleError(Exception): def __init__(self, *args: Any, **kwargs: Any) -> None: ensure_style_detected() - super().__init__(self.message.format(*args, **STYLES, **kwargs)) + message = kwargs.pop("message", self.message) + super().__init__(message.format(*args, **STYLES, **kwargs)) class NeptuneScaleWarning(Warning): @@ -62,6 +63,25 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(self.message.format(*args, **STYLES, **kwargs)) +class NeptuneBadRequestError(NeptuneScaleError): + """ + A generic "bad request" error. Pass `reason` to provide a custom message. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + reason = kwargs.get("reason", None) + if not reason: + reason = "The request contains invalid data" + kwargs["reason"] = reason + kwargs["status_code"] = kwargs.get("status_code", 400) + super().__init__(*args, **kwargs) + + message = """ +{h1} +NeptuneBadRequestError({status_code}): {reason} +""" + + class NeptuneSynchronizationStopped(NeptuneScaleError): message = "Internal synchronization process was stopped." @@ -206,6 +226,14 @@ class NeptuneProjectInvalidName(NeptuneScaleError): """ +class NeptuneProjectAlreadyExists(NeptuneScaleError): + message = """ +{h1} +NeptuneProjectAlreadyExists: A project with the provided name or project key already exists. +{end} +""" + + class NeptuneRunNotFound(NeptuneScaleError): message = """ {h1} diff --git a/src/neptune_scale/net/api_client.py b/src/neptune_scale/net/api_client.py index baa35e15..0b640ab9 100644 --- a/src/neptune_scale/net/api_client.py +++ b/src/neptune_scale/net/api_client.py @@ -15,7 +15,7 @@ # from __future__ import annotations -__all__ = ("HostedApiClient", "MockedApiClient", "ApiClient", "backend_factory") +__all__ = ("HostedApiClient", "MockedApiClient", "ApiClient", "backend_factory", "with_api_errors_handling") import abc import os @@ -24,9 +24,11 @@ from http import HTTPStatus from typing import ( Any, + Callable, Literal, ) +import httpx from httpx import Timeout from neptune_api import ( AuthenticatedClient, @@ -39,6 +41,13 @@ ) from neptune_api.auth_helpers import exchange_api_key from neptune_api.credentials import Credentials +from neptune_api.errors import ( + InvalidApiTokenException, + UnableToDeserializeApiKeyError, + UnableToExchangeApiKeyError, + UnableToRefreshTokenError, + UnexpectedStatus, +) from neptune_api.models import ( ClientConfig, Error, @@ -55,6 +64,11 @@ from neptune_api.proto.neptune_pb.ingest.v1.pub.request_status_pb2 import RequestStatus from neptune_api.types import Response +from neptune_scale.exceptions import ( + NeptuneConnectionLostError, + NeptuneInvalidCredentialsError, + NeptuneUnableToAuthenticateError, +) from neptune_scale.sync.parameters import REQUEST_TIMEOUT from neptune_scale.util.abstract import Resource from neptune_scale.util.envs import ALLOW_SELF_SIGNED_CERTIFICATE @@ -123,24 +137,24 @@ def __init__(self, api_token: str) -> None: logger.debug("Trying to connect to Neptune API") config, token_urls = get_config_and_token_urls(credentials=credentials, verify_ssl=verify_ssl) - self._backend = create_auth_api_client( + self.backend = create_auth_api_client( credentials=credentials, config=config, token_refreshing_urls=token_urls, verify_ssl=verify_ssl ) logger.debug("Connected to Neptune API") def submit(self, operation: RunOperation, family: str) -> Response[SubmitResponse]: - return submit_operation.sync_detailed(client=self._backend, body=operation, family=family) + return submit_operation.sync_detailed(client=self.backend, body=operation, family=family) def check_batch(self, request_ids: list[str], project: str) -> Response[BulkRequestStatus]: return check_request_status_bulk.sync_detailed( - client=self._backend, + client=self.backend, project_identifier=project, body=RequestIdList(ids=[RequestId(value=request_id) for request_id in request_ids]), ) def close(self) -> None: logger.debug("Closing API client") - self._backend.__exit__() + self.backend.__exit__() class MockedApiClient(ApiClient): @@ -171,3 +185,19 @@ def backend_factory(api_token: str, mode: Literal["async", "disabled"]) -> ApiCl if mode == "disabled": return MockedApiClient() return HostedApiClient(api_token=api_token) + + +def with_api_errors_handling(func: Callable[..., Any]) -> Callable[..., Any]: + def wrapper(*args: Any, **kwargs: Any) -> Any: + try: + return func(*args, **kwargs) + except (InvalidApiTokenException, UnableToDeserializeApiKeyError): + raise NeptuneInvalidCredentialsError() + except (UnableToRefreshTokenError, UnableToExchangeApiKeyError, UnexpectedStatus): + raise NeptuneUnableToAuthenticateError() + except (httpx.ConnectError, httpx.TimeoutException, httpx.RemoteProtocolError): + raise NeptuneConnectionLostError() + except Exception as e: + raise e + + return wrapper diff --git a/src/neptune_scale/net/projects.py b/src/neptune_scale/net/projects.py new file mode 100644 index 00000000..999280af --- /dev/null +++ b/src/neptune_scale/net/projects.py @@ -0,0 +1,84 @@ +import os +import re +from enum import Enum +from json import JSONDecodeError +from typing import ( + Any, + Optional, +) + +import httpx + +from neptune_scale.exceptions import ( + NeptuneApiTokenNotProvided, + NeptuneBadRequestError, + NeptuneProjectAlreadyExists, +) +from neptune_scale.net.api_client import ( + HostedApiClient, + with_api_errors_handling, +) +from neptune_scale.util.envs import API_TOKEN_ENV_NAME + +PROJECTS_PATH_BASE = "/api/backend/v1/projects" + + +class ProjectVisibility(Enum): + PRIVATE = "priv" + PUBLIC = "pub" + WORKSPACE = "workspace" + + +ORGANIZATION_NOT_FOUND_RE = re.compile(r"Organization .* not found") + + +@with_api_errors_handling +def create_project( + workspace: str, + name: str, + *, + visibility: ProjectVisibility = ProjectVisibility.PRIVATE, + description: Optional[str] = None, + key: Optional[str] = None, + fail_if_exists: bool = False, + api_token: Optional[str] = None, +) -> None: + api_token = api_token or os.environ.get(API_TOKEN_ENV_NAME) + if api_token is None: + raise NeptuneApiTokenNotProvided() + + client = HostedApiClient(api_token=api_token) + visibility = ProjectVisibility(visibility) + + body = { + "name": name, + "description": description, + "projectKey": key, + "organizationIdentifier": workspace, + "visibility": visibility.value, + } + + response = client.backend.get_httpx_client().request("post", PROJECTS_PATH_BASE, json=body) + json = _safe_json(response) + + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + code = e.response.status_code + if code == 409: + if fail_if_exists: + raise NeptuneProjectAlreadyExists() + # We need to match plain text, as this is what the backend returns + elif code == 404 and ORGANIZATION_NOT_FOUND_RE.match(response.text): + raise NeptuneBadRequestError(status_code=code, reason=f"Workspace '{workspace}' not found") + elif code // 100 == 4: + raise NeptuneBadRequestError(status_code=code, reason=json.get("message")) + else: + raise e + + +def _safe_json(response: httpx.Response) -> Any: + try: + return response.json() + except JSONDecodeError: + return {} diff --git a/src/neptune_scale/projects.py b/src/neptune_scale/projects.py new file mode 100644 index 00000000..e6bbc3f6 --- /dev/null +++ b/src/neptune_scale/projects.py @@ -0,0 +1,117 @@ +import re +from typing import ( + Optional, + Tuple, + cast, +) + +from neptune_scale.api.validation import verify_type +from neptune_scale.net import projects +from neptune_scale.net.projects import ProjectVisibility + +PROJECT_QUALIFIED_NAME_RE = re.compile(r"^((?P[\w\-.]+)/)?(?P[\w\-.]+)$") + + +def create_project( + name: str, + *, + workspace: Optional[str] = None, + visibility: str = ProjectVisibility.PRIVATE.value, + description: Optional[str] = None, + key: Optional[str] = None, + fail_if_exists: bool = False, + api_token: Optional[str] = None, +) -> str: + """Creates a new project in a Neptune workspace. + + Args: + name (str): Name of the project. Can contain letters and hyphens (-). For example, "project-x". + workspace (str, optional): Name of your Neptune workspace. + You can omit this argument if you include the workspace name in the `name` argument. + visibility: Level of privacy for the project. Options: + - "pub": Public. Anyone on the internet can see it. + - "priv" (default): Private. Only users specifically assigned to the project can access it. Requires a plan with + project-level access control. + - "workspace" (team workspaces only): Accessible to all workspace members. + description: Project description. If None, it's left empty. + key: Project identifier. Must contain 1-10 upper case letters or numbers (at least one letter). + For example, "PX2". If you leave it out, Neptune generates a project key for you. + fail_if_exists: If the project already exists and this flag is set to `True`, an error is raised. + api_token: Account's API token. + If not provided, the value of the NEPTUNE_API_TOKEN environment variable is used (recommended). + + Returns: + The name of the new project created. + """ + + verify_type("name", name, str) + verify_type("workspace", workspace, (str, type(None))) + verify_type("visibility", visibility, str) + verify_type("description", description, (str, type(None))) + verify_type("key", key, (str, type(None))) + verify_type("api_token", api_token, (str, type(None))) + + workspace, name = extract_workspace_and_project(name=name, workspace=workspace) + projects.create_project( + workspace=workspace, + name=name, + visibility=visibility, + description=description, + key=key, + fail_if_exists=fail_if_exists, + api_token=api_token, + ) + + return normalize_project_name(name, workspace) + + +def extract_workspace_and_project(name: str, workspace: Optional[str] = None) -> Tuple[str, str]: + """Return a tuple of (workspace name, project name) from the provided + fully qualified project name, or a name + workspace + + >>> extract_workspace_and_project("my-own.workspace_/pr_oj-ect") + ('my-own.workspace_', 'pr_oj-ect') + >>> extract_workspace_and_project("project", "workspace") + ('workspace', 'project') + >>> extract_workspace_and_project("workspace/project", "workspace") + ('workspace', 'project') + >>> extract_workspace_and_project("workspace/project", "another_workspace") + Traceback (most recent call last): + ... + ValueError: The provided `workspace` argument `another_workspace` is different ... + >>> extract_workspace_and_project("project") + Traceback (most recent call last): + ... + ValueError: Workspace not provided ... + >>> extract_workspace_and_project("workspace/project!@#") + Traceback (most recent call last): + ... + ValueError: Invalid project name ... + """ + project_spec = PROJECT_QUALIFIED_NAME_RE.search(name) + + if not project_spec: + raise ValueError(f"Invalid project name `{name}`") + + extracted_workspace, extracted_project_name = ( + project_spec["workspace"], + project_spec["project"], + ) + + if not workspace and not extracted_workspace: + raise ValueError("Workspace not provided in neither project name or the `workspace` parameter.") + + if workspace and extracted_workspace and workspace != extracted_workspace: + raise ValueError( + f"The provided `workspace` argument `{workspace}` is different from the one in project name `{name}`" + ) + + final_workspace_name = cast(str, extracted_workspace or workspace) + + return final_workspace_name, extracted_project_name + + +def normalize_project_name(name: str, workspace: Optional[str] = None) -> str: + extracted_workspace_name, extracted_project_name = extract_workspace_and_project(name=name, workspace=workspace) + + return f"{extracted_workspace_name}/{extracted_project_name}" diff --git a/src/neptune_scale/sync/sync_process.py b/src/neptune_scale/sync/sync_process.py index 501e3ba0..cbe373e6 100644 --- a/src/neptune_scale/sync/sync_process.py +++ b/src/neptune_scale/sync/sync_process.py @@ -12,8 +12,6 @@ ) from types import FrameType from typing import ( - Any, - Callable, Dict, Generic, List, @@ -25,14 +23,6 @@ ) import backoff -import httpx -from neptune_api.errors import ( - InvalidApiTokenException, - UnableToDeserializeApiKeyError, - UnableToExchangeApiKeyError, - UnableToRefreshTokenError, - UnexpectedStatus, -) from neptune_api.proto.google_rpc.code_pb2 import Code from neptune_api.proto.neptune_pb.ingest.v1.ingest_pb2 import IngestCode from neptune_api.proto.neptune_pb.ingest.v1.pub.client_pb2 import ( @@ -51,7 +41,6 @@ NeptuneConnectionLostError, NeptuneFloatValueNanInfUnsupported, NeptuneInternalServerError, - NeptuneInvalidCredentialsError, NeptuneOperationsQueueMaxSizeExceeded, NeptuneProjectInvalidName, NeptuneProjectNotFound, @@ -68,7 +57,6 @@ NeptuneStringSetExceedsSizeLimit, NeptuneStringValueExceedsSizeLimit, NeptuneSynchronizationStopped, - NeptuneUnableToAuthenticateError, NeptuneUnauthorizedError, NeptuneUnexpectedError, NeptuneUnexpectedResponseError, @@ -76,6 +64,7 @@ from neptune_scale.net.api_client import ( ApiClient, backend_factory, + with_api_errors_handling, ) from neptune_scale.sync.aggregating_queue import AggregatingQueue from neptune_scale.sync.errors_tracking import ErrorsQueue @@ -175,22 +164,6 @@ def commit(self, n: int) -> None: self._queue.get() -def with_api_errors_handling(func: Callable[..., Any]) -> Callable[..., Any]: - def wrapper(*args: Any, **kwargs: Any) -> Any: - try: - return func(*args, **kwargs) - except (InvalidApiTokenException, UnableToDeserializeApiKeyError): - raise NeptuneInvalidCredentialsError() - except (UnableToRefreshTokenError, UnableToExchangeApiKeyError, UnexpectedStatus): - raise NeptuneUnableToAuthenticateError() - except (httpx.ConnectError, httpx.TimeoutException, httpx.RemoteProtocolError): - raise NeptuneConnectionLostError() - except Exception as e: - raise e - - return wrapper - - class SyncProcess(Process): def __init__( self,