From ebf9fc05c743fa728b65471506a7e7e3098e8cf8 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 14 Oct 2023 15:49:59 +0100 Subject: [PATCH 01/31] WIP: Lazy entry points in verdi --- aiida/cmdline/params/types/identifier.py | 31 ++++++++++++++---------- aiida/cmdline/params/types/plugin.py | 7 ++++++ aiida/plugins/entry_point.py | 24 +++++++++++------- aiida/plugins/factories.py | 5 ++-- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index fc15539fdd..c5b5091849 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -51,19 +51,21 @@ def __init__(self, sub_classes=None): self._sub_classes = None self._entry_points = [] - if sub_classes is not None: - - if not isinstance(sub_classes, tuple): - raise TypeError('sub_classes should be a tuple of entry point strings') - - for entry_point_string in sub_classes: - - try: - entry_point = get_entry_point_from_string(entry_point_string) - except (ValueError, exceptions.EntryPointError) as exception: - raise ValueError(f'{entry_point_string} is not a valid entry point string: {exception}') - else: - self._entry_points.append(entry_point) + if sub_classes is None: + return + if not isinstance(sub_classes, tuple): + raise TypeError('sub_classes should be a tuple of entry point strings') + + self.sub_classes = sub_classes + # TODO: Add a property that loads all this on demand + return + for entry_point_string in sub_classes: + try: + entry_point = get_entry_point_from_string(entry_point_string) + except (ValueError, exceptions.EntryPointError) as exception: + raise ValueError(f'{entry_point_string} is not a valid entry point string: {exception}') + else: + self._entry_points.append(entry_point) @property @abstractmethod @@ -89,6 +91,9 @@ def convert(self, value, param, ctx): from aiida.common import exceptions from aiida.orm.utils.loaders import OrmEntityLoader + # TODO: Remove this + # raise ValueError("") + value = super().convert(value, param, ctx) if not value: diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index d0f684fc80..1cd0b1abf8 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -68,6 +68,10 @@ def __init__(self, group=None, load=False, *args, **kwargs): is not specified use the tuple of all recognized entry point groups. """ # pylint: disable=keyword-arg-before-vararg + self.load = load + self._groups = None + # TODO: Implement the following logic lazily + return valid_entry_point_groups = get_entry_point_groups() if group is None: @@ -105,11 +109,14 @@ def _init_entry_points(self): once in the constructor after setting self.groups because the groups should not be changed after instantiation """ + # TODO: Optimize this self._entry_points = [(group, entry_point) for group in self.groups for entry_point in get_entry_points(group)] self._entry_point_names = [entry_point.name for group in self.groups for entry_point in get_entry_points(group)] @property def groups(self): + # TODO: Remove this + # raise ValueError("Whoops groups") return self._groups @property diff --git a/aiida/plugins/entry_point.py b/aiida/plugins/entry_point.py index 168c38490d..60328abdcf 100644 --- a/aiida/plugins/entry_point.py +++ b/aiida/plugins/entry_point.py @@ -13,19 +13,19 @@ import enum import functools import traceback -from typing import Any, List, Optional, Sequence, Set, Tuple - -# importlib.metadata was introduced into the standard library in python 3.8, -# but was then updated in python 3.10 to use an improved API. -# So for now we use the backport importlib_metadata package. -from importlib_metadata import EntryPoint, EntryPoints -from importlib_metadata import entry_points as _eps +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Set, Tuple from aiida.common.exceptions import LoadingEntryPointError, MissingEntryPointError, MultipleEntryPointError from aiida.common.warnings import warn_deprecation from . import factories +if TYPE_CHECKING: + # importlib.metadata was introduced into the standard library in python 3.8, + # but was then updated in python 3.10 to use an improved API. + # So for now we use the backport importlib_metadata package. + from importlib_metadata import EntryPoint, EntryPoints + __all__ = ('load_entry_point', 'load_entry_point_from_string', 'parse_entry_point', 'get_entry_points') ENTRY_POINT_GROUP_PREFIX = 'aiida.' @@ -43,8 +43,10 @@ def eps() -> EntryPoints: which will always iterate over all entry points since it looks for possible duplicate entries. """ - entry_points = _eps() - return EntryPoints(sorted(entry_points, key=lambda x: x.group)) + # raise ValueError("eps!") + from importlib_metadata import EntryPoints, entry_points + all_eps = entry_points() + return EntryPoints(sorted(all_eps, key=lambda x: x.group)) @functools.lru_cache(maxsize=100) @@ -53,6 +55,9 @@ def eps_select(group: str, name: str | None = None) -> EntryPoints: A thin wrapper around entry_points.select() calls, which are expensive so we want to cache them. """ + # msg = f"{group=} {name=}" + # print(msg) + # raise ValueError(msg) if name is None: return eps().select(group=group) return eps().select(group=group, name=name) @@ -132,6 +137,7 @@ class EntryPointFormat(enum.Enum): def parse_entry_point(group: str, spec: str) -> EntryPoint: """Return an entry point, given its group and spec (as formatted in the setup)""" + from importlib_metadata import EntryPoint name, value = spec.split('=', maxsplit=1) return EntryPoint(group=group, name=name.strip(), value=value.strip()) diff --git a/aiida/plugins/factories.py b/aiida/plugins/factories.py index 80109c7a1f..5c11a7c4c9 100644 --- a/aiida/plugins/factories.py +++ b/aiida/plugins/factories.py @@ -9,11 +9,11 @@ ########################################################################### # pylint: disable=invalid-name,cyclic-import """Definition of factories to load classes from the various plugin groups.""" +from __future__ import annotations + from inspect import isclass from typing import TYPE_CHECKING, Any, Callable, Literal, NoReturn, Tuple, Type, Union, overload -from importlib_metadata import EntryPoint - from aiida.common.exceptions import InvalidEntryPointTypeError __all__ = ( @@ -22,6 +22,7 @@ ) if TYPE_CHECKING: + from importlib_metadata import EntryPoint from aiida.engine import CalcJob, CalcJobImporter, WorkChain from aiida.orm import Data, Group from aiida.orm.implementation import StorageBackend From 3b5dde91812b69d575e2e2656f8ef3c7706c3dee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:30:58 +0000 Subject: [PATCH 02/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiida/plugins/factories.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiida/plugins/factories.py b/aiida/plugins/factories.py index 5c11a7c4c9..8b583f1527 100644 --- a/aiida/plugins/factories.py +++ b/aiida/plugins/factories.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from importlib_metadata import EntryPoint + from aiida.engine import CalcJob, CalcJobImporter, WorkChain from aiida.orm import Data, Group from aiida.orm.implementation import StorageBackend From 8559c370e5165e59e35084aa26a5f62752ecb447 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 17 Oct 2023 19:22:30 +0100 Subject: [PATCH 03/31] Try running tests --- aiida/cmdline/params/types/identifier.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index c5b5091849..5474505f74 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -58,6 +58,7 @@ def __init__(self, sub_classes=None): self.sub_classes = sub_classes # TODO: Add a property that loads all this on demand + self._entry_points = [] return for entry_point_string in sub_classes: try: From 47aa3ba216a185d782bee5ccfe47c4435cd17313 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 17 Oct 2023 23:13:25 +0100 Subject: [PATCH 04/31] Fix EntryPoint imports --- aiida/orm/groups.py | 3 ++- aiida/orm/nodes/node.py | 2 +- tests/plugins/test_entry_point.py | 9 +++------ tests/test_conftest.py | 3 ++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/aiida/orm/groups.py b/aiida/orm/groups.py index 83adb822c8..7f7c4daaff 100644 --- a/aiida/orm/groups.py +++ b/aiida/orm/groups.py @@ -20,9 +20,10 @@ from . import convert, entities, extras, users if TYPE_CHECKING: + from importlib_metadata import EntryPoint + from aiida.orm import Node, User from aiida.orm.implementation import BackendGroup, StorageBackend - from aiida.plugins.entry_point import EntryPoint # type: ignore[attr-defined] __all__ = ('Group', 'AutoGroup', 'ImportGroup', 'UpfFamily') diff --git a/aiida/orm/nodes/node.py b/aiida/orm/nodes/node.py index a5d7400607..afa4c5f5a4 100644 --- a/aiida/orm/nodes/node.py +++ b/aiida/orm/nodes/node.py @@ -39,7 +39,7 @@ from .repository import NodeRepository if TYPE_CHECKING: - from aiida.plugins.entry_point import EntryPoint # type: ignore[attr-defined] + from importlib_metadata import EntryPoint from ..implementation import BackendNode, StorageBackend diff --git a/tests/plugins/test_entry_point.py b/tests/plugins/test_entry_point.py index 70fdd9fda4..9952e6d6a3 100644 --- a/tests/plugins/test_entry_point.py +++ b/tests/plugins/test_entry_point.py @@ -9,17 +9,14 @@ ########################################################################### # pylint: disable=redefined-outer-name """Tests for the :mod:`~aiida.plugins.entry_point` module.""" +from importlib_metadata import EntryPoint as EP +from importlib_metadata import EntryPoints import pytest from aiida.common.exceptions import MissingEntryPointError, MultipleEntryPointError from aiida.common.warnings import AiidaDeprecationWarning from aiida.plugins import entry_point -from aiida.plugins.entry_point import ( # type: ignore[attr-defined] - EntryPoints, - get_entry_point, - validate_registered_entry_points, -) -from aiida.plugins.entry_point import EntryPoint as EP # type: ignore[attr-defined] +from aiida.plugins.entry_point import get_entry_point, validate_registered_entry_points def test_validate_registered_entry_points(): diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 5c35204cb0..52460fff38 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """Tests for fixtures in the ``conftest.py``.""" +from importlib_metadata import EntryPoint import pytest from aiida.common.exceptions import MissingEntryPointError -from aiida.plugins.entry_point import EntryPoint, get_entry_point, load_entry_point # type: ignore[attr-defined] +from aiida.plugins.entry_point import get_entry_point, load_entry_point ENTRY_POINT_GROUP = 'aiida.calculations.importers' From aebdb2ad1f68306b22133a7ed867a5bec4513659 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Tue, 17 Oct 2023 23:29:04 +0100 Subject: [PATCH 05/31] Revert breaking changes to see if tests pass --- aiida/cmdline/params/types/identifier.py | 11 ++++++----- aiida/cmdline/params/types/plugin.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index 5474505f74..053165c7ea 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -10,6 +10,8 @@ """ Module for custom click param type identifier """ +from __future__ import annotations + from abc import ABC, abstractmethod import click @@ -30,7 +32,7 @@ class IdentifierParamType(click.ParamType, ABC): which should be a subclass of `aiida.orm.utils.loaders.OrmEntityLoader` for the corresponding orm class. """ - def __init__(self, sub_classes=None): + def __init__(self, sub_classes: tuple | None = None): """ Construct the parameter type, optionally specifying a tuple of entry points that reference classes that should be a sub class of the base orm class of the orm class loader. The classes pointed to by @@ -48,18 +50,17 @@ def __init__(self, sub_classes=None): """ from aiida.common import exceptions - self._sub_classes = None + self._sub_classes = sub_classes self._entry_points = [] if sub_classes is None: return + if not isinstance(sub_classes, tuple): raise TypeError('sub_classes should be a tuple of entry point strings') - self.sub_classes = sub_classes # TODO: Add a property that loads all this on demand - self._entry_points = [] - return + # return for entry_point_string in sub_classes: try: entry_point = get_entry_point_from_string(entry_point_string) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 1cd0b1abf8..c68a0d08cb 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -71,14 +71,14 @@ def __init__(self, group=None, load=False, *args, **kwargs): self.load = load self._groups = None # TODO: Implement the following logic lazily - return + # return valid_entry_point_groups = get_entry_point_groups() if group is None: self._groups = tuple(valid_entry_point_groups) else: if isinstance(group, str): - invalidated_groups = tuple([group]) + invalidated_groups = (group,) elif isinstance(group, tuple): invalidated_groups = group else: From 82cefcd0570aa7ea0f02ddc8acf1c3d6a1139199 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 01:31:05 +0100 Subject: [PATCH 06/31] I guess this does not work? --- aiida/cmdline/params/types/identifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index 053165c7ea..ddb1a68092 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -32,7 +32,7 @@ class IdentifierParamType(click.ParamType, ABC): which should be a subclass of `aiida.orm.utils.loaders.OrmEntityLoader` for the corresponding orm class. """ - def __init__(self, sub_classes: tuple | None = None): + def __init__(self, sub_classes: tuple[str] | None = None): """ Construct the parameter type, optionally specifying a tuple of entry points that reference classes that should be a sub class of the base orm class of the orm class loader. The classes pointed to by @@ -50,7 +50,7 @@ def __init__(self, sub_classes: tuple | None = None): """ from aiida.common import exceptions - self._sub_classes = sub_classes + self._sub_classes: tuple | None = None self._entry_points = [] if sub_classes is None: From a0dd26fe647f3383458f98d0d7f1e64735daeeec Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 02:15:20 +0100 Subject: [PATCH 07/31] One more --- aiida/manage/tests/pytest_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiida/manage/tests/pytest_fixtures.py b/aiida/manage/tests/pytest_fixtures.py index fe0efd307b..8f8a0efabe 100644 --- a/aiida/manage/tests/pytest_fixtures.py +++ b/aiida/manage/tests/pytest_fixtures.py @@ -32,7 +32,7 @@ import uuid import warnings -from importlib_metadata import EntryPoints +from importlib_metadata import EntryPoint, EntryPoints import plumpy import pytest import wrapt @@ -822,7 +822,7 @@ def add( value = f'{value.__module__}:{value.__name__}' group, name = self._validate_entry_point(entry_point_string, group, name) - entry_point = plugins.entry_point.EntryPoint(name, value, group) + entry_point = EntryPoint(name, value, group) self.entry_points = EntryPoints(self.entry_points + (entry_point,)) def remove( From 15b498f5621bd049df59fe86804c999bbdcae978 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 04:19:26 +0100 Subject: [PATCH 08/31] Break --- aiida/cmdline/params/types/identifier.py | 2 +- aiida/cmdline/params/types/plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index ddb1a68092..734c4da289 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -60,7 +60,7 @@ def __init__(self, sub_classes: tuple[str] | None = None): raise TypeError('sub_classes should be a tuple of entry point strings') # TODO: Add a property that loads all this on demand - # return + return for entry_point_string in sub_classes: try: entry_point = get_entry_point_from_string(entry_point_string) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index c68a0d08cb..6d55eb4883 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -11,7 +11,6 @@ import functools import click -from importlib_metadata import EntryPoint from aiida.common import exceptions from aiida.plugins import factories @@ -71,7 +70,7 @@ def __init__(self, group=None, load=False, *args, **kwargs): self.load = load self._groups = None # TODO: Implement the following logic lazily - # return + return valid_entry_point_groups = get_entry_point_groups() if group is None: @@ -243,6 +242,7 @@ def convert(self, value, param, ctx): Convert the string value to an entry point instance, if the value can be successfully parsed into an actual entry point. Will raise click.BadParameter if validation fails. """ + from importlib_metadata import EntryPoint # If the value is already of the expected return type, simply return it. This behavior is new in `click==8.0`: # https://click.palletsprojects.com/en/8.0.x/parameters/#implementing-custom-types if isinstance(value, EntryPoint): From 2d6b8b0b622c646762b39aa1c4626ddaf49217e5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 03:20:43 +0000 Subject: [PATCH 09/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiida/cmdline/params/types/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 6d55eb4883..7b388c3ef9 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -243,6 +243,7 @@ def convert(self, value, param, ctx): into an actual entry point. Will raise click.BadParameter if validation fails. """ from importlib_metadata import EntryPoint + # If the value is already of the expected return type, simply return it. This behavior is new in `click==8.0`: # https://click.palletsprojects.com/en/8.0.x/parameters/#implementing-custom-types if isinstance(value, EntryPoint): From 9e01860a93501bff2b62dfb677291b6a73be0e04 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 04:32:00 +0100 Subject: [PATCH 10/31] Remove click from aiida.orm --- aiida/orm/nodes/data/code/abstract.py | 6 ++++-- aiida/orm/nodes/data/code/containerized.py | 4 ++-- aiida/orm/nodes/data/code/installed.py | 6 ++++-- aiida/orm/nodes/data/code/portable.py | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/aiida/orm/nodes/data/code/abstract.py b/aiida/orm/nodes/data/code/abstract.py index 721049879f..5a2f4ec0e2 100644 --- a/aiida/orm/nodes/data/code/abstract.py +++ b/aiida/orm/nodes/data/code/abstract.py @@ -15,8 +15,6 @@ import pathlib from typing import TYPE_CHECKING -import click - from aiida.common import exceptions from aiida.common.folders import Folder from aiida.common.lang import type_check @@ -309,6 +307,8 @@ def get_builder(self) -> 'ProcessBuilder': @staticmethod def cli_validate_label_uniqueness(_, __, value): """Validate the uniqueness of the label of the code.""" + import click + from aiida.orm import load_code try: @@ -330,6 +330,8 @@ def get_cli_options(cls) -> collections.OrderedDict: @classmethod def _get_cli_options(cls) -> dict: """Return the CLI options that would allow to create an instance of this class.""" + import click + from aiida.cmdline.params.options.interactive import TemplateInteractiveOption return { diff --git a/aiida/orm/nodes/data/code/containerized.py b/aiida/orm/nodes/data/code/containerized.py index afb82e76d9..99230434b4 100644 --- a/aiida/orm/nodes/data/code/containerized.py +++ b/aiida/orm/nodes/data/code/containerized.py @@ -16,8 +16,6 @@ import pathlib -import click - from aiida.common.lang import type_check from .installed import InstalledCode @@ -108,6 +106,8 @@ def get_prepend_cmdline_params( @classmethod def _get_cli_options(cls) -> dict: """Return the CLI options that would allow to create an instance of this class.""" + import click + options = { 'engine_command': { 'short_name': '-E', diff --git a/aiida/orm/nodes/data/code/installed.py b/aiida/orm/nodes/data/code/installed.py index 6b0c3397ce..9379c15eef 100644 --- a/aiida/orm/nodes/data/code/installed.py +++ b/aiida/orm/nodes/data/code/installed.py @@ -18,8 +18,6 @@ import pathlib -import click - from aiida.common import exceptions from aiida.common.lang import type_check from aiida.common.log import override_log_level @@ -155,6 +153,8 @@ def filepath_executable(self, value: str) -> None: @staticmethod def cli_validate_label_uniqueness(ctx, _, value): """Validate the uniqueness of the label of the code.""" + import click + from aiida.orm import load_code computer = ctx.params.get('computer', None) @@ -178,6 +178,8 @@ def cli_validate_label_uniqueness(ctx, _, value): @classmethod def _get_cli_options(cls) -> dict: """Return the CLI options that would allow to create an instance of this class.""" + import click + from aiida.cmdline.params.types import ComputerParamType options = { diff --git a/aiida/orm/nodes/data/code/portable.py b/aiida/orm/nodes/data/code/portable.py index 8fb2bd8364..6fac593818 100644 --- a/aiida/orm/nodes/data/code/portable.py +++ b/aiida/orm/nodes/data/code/portable.py @@ -21,8 +21,6 @@ import pathlib -import click - from aiida.common import exceptions from aiida.common.folders import Folder from aiida.common.lang import type_check @@ -148,6 +146,8 @@ def filepath_executable(self, value: str) -> None: @classmethod def _get_cli_options(cls) -> dict: """Return the CLI options that would allow to create an instance of this class.""" + import click + options = { 'filepath_executable': { 'short_name': '-X', From a66ba9cc9dd05c94fcf3d85931a0a989c49df2de Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 12:53:03 +0100 Subject: [PATCH 11/31] initialize to empty lists --- aiida/cmdline/params/types/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 7b388c3ef9..f21683e6e5 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -69,6 +69,8 @@ def __init__(self, group=None, load=False, *args, **kwargs): # pylint: disable=keyword-arg-before-vararg self.load = load self._groups = None + self._entry_points = [] + self._entry_point_names = [] # TODO: Implement the following logic lazily return valid_entry_point_groups = get_entry_point_groups() From 2f8697a3609d3fbc34854e6e565c9d666971cec3 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 13:24:23 +0100 Subject: [PATCH 12/31] Lazy import tabulate, ~6ms --- aiida/cmdline/__init__.py | 1 + aiida/cmdline/commands/cmd_code.py | 7 +++---- aiida/cmdline/commands/cmd_computer.py | 7 +++---- aiida/cmdline/commands/cmd_node.py | 17 +++++++++-------- aiida/cmdline/commands/cmd_rabbitmq.py | 5 ++--- aiida/cmdline/utils/__init__.py | 1 + aiida/cmdline/utils/common.py | 7 ++++++- aiida/cmdline/utils/echo.py | 12 ++++++++++-- 8 files changed, 35 insertions(+), 22 deletions(-) diff --git a/aiida/cmdline/__init__.py b/aiida/cmdline/__init__.py index f34e1f5f52..4b3e166a76 100644 --- a/aiida/cmdline/__init__.py +++ b/aiida/cmdline/__init__.py @@ -53,6 +53,7 @@ 'echo_info', 'echo_report', 'echo_success', + 'echo_tabulate', 'echo_warning', 'format_call_graph', 'is_verbose', diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index 305b1078e3..eb4cc1ea7c 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -12,13 +12,12 @@ from functools import partial import click -import tabulate from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup from aiida.cmdline.params import arguments, options, types from aiida.cmdline.params.options.commands import code as options_code -from aiida.cmdline.utils import echo +from aiida.cmdline.utils import echo, echo_tabulate from aiida.cmdline.utils.decorators import deprecated_command, with_dbenv from aiida.common import exceptions @@ -232,7 +231,7 @@ def show(code): if is_verbose(): table.append(['Calculations', len(code.base.links.get_outgoing().all())]) - echo.echo(tabulate.tabulate(table)) + echo_tabulate(table) @verdi_code.command() @@ -419,7 +418,7 @@ def code_list(computer, default_calc_job_plugin, all_entries, all_users, raw, sh row.append('@'.join(str(result[entity][projection]) for entity, projection in VALID_PROJECTIONS[key])) table.append(row) - echo.echo(tabulate.tabulate(table, headers=headers, tablefmt=table_format)) + echo_tabulate(table, headers=headers, tablefmt=table_format) if not raw: echo.echo_report('\nUse `verdi code show IDENTIFIER` to see details for a code', prefix=False) diff --git a/aiida/cmdline/commands/cmd_computer.py b/aiida/cmdline/commands/cmd_computer.py index 182d8ecc52..27fba6bfc9 100644 --- a/aiida/cmdline/commands/cmd_computer.py +++ b/aiida/cmdline/commands/cmd_computer.py @@ -14,12 +14,11 @@ from math import isclose import click -import tabulate from aiida.cmdline.commands.cmd_verdi import VerdiCommandGroup, verdi from aiida.cmdline.params import arguments, options from aiida.cmdline.params.options.commands import computer as options_computer -from aiida.cmdline.utils import echo +from aiida.cmdline.utils import echo, echo_tabulate from aiida.cmdline.utils.decorators import with_dbenv from aiida.common.exceptions import EntryPointError, ValidationError from aiida.plugins.entry_point import get_entry_point_names @@ -461,7 +460,7 @@ def computer_show(computer): ['Prepend text', computer.get_prepend_text()], ['Append text', computer.get_append_text()], ] - echo.echo(tabulate.tabulate(table)) + echo_tabulate(table) @verdi_computer.command('relabel') @@ -697,4 +696,4 @@ def computer_config_show(computer, user, defaults, as_option_string): table.append((f'* {name}', config[name])) else: table.append((f'* {name}', '-')) - echo.echo(tabulate.tabulate(table, tablefmt='plain')) + echo_tabulate(table, tablefmt='plain') diff --git a/aiida/cmdline/commands/cmd_node.py b/aiida/cmdline/commands/cmd_node.py index 6af3eee930..728c27b185 100644 --- a/aiida/cmdline/commands/cmd_node.py +++ b/aiida/cmdline/commands/cmd_node.py @@ -9,15 +9,13 @@ ########################################################################### """`verdi node` command.""" import pathlib -import shutil import click -import tabulate from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.params import arguments, options from aiida.cmdline.params.types.plugin import PluginParamType -from aiida.cmdline.utils import decorators, echo, multi_line_input +from aiida.cmdline.utils import decorators, echo, echo_tabulate, multi_line_input from aiida.cmdline.utils.decorators import with_dbenv from aiida.common import exceptions, timezone from aiida.common.links import GraphTraversalRules @@ -43,6 +41,7 @@ def repo_cat(node, relative_path): For ``SinglefileData`` nodes, the `RELATIVE_PATH` does not have to be specified as it is determined automatically. """ import errno + import shutil import sys from aiida.orm import SinglefileData @@ -92,6 +91,8 @@ def repo_dump(node, output_directory): The output directory should not exist. If it does, the command will abort. """ + import shutil + from aiida.repository import FileType output_directory = pathlib.Path(output_directory) @@ -146,9 +147,9 @@ def node_label(nodes, label, raw, force): table.append([node.pk, node.label]) if raw: - echo.echo(tabulate.tabulate(table, tablefmt='plain')) + echo_tabulate(table, tablefmt='plain') else: - echo.echo(tabulate.tabulate(table, headers=['ID', 'Label'])) + echo_tabulate(table, headers=['ID', 'Label']) else: if not force: @@ -180,9 +181,9 @@ def node_description(nodes, description, force, raw): table.append([node.pk, node.description]) if raw: - echo.echo(tabulate.tabulate(table, tablefmt='plain')) + echo_tabulate(table, tablefmt='plain') else: - echo.echo(tabulate.tabulate(table, headers=['ID', 'Description'])) + echo_tabulate(table, headers=['ID', 'Description']) else: if not force: @@ -229,7 +230,7 @@ def node_show(nodes, print_groups): table = [(gr['groups']['id'], gr['groups']['label'], gr['groups']['type_string']) for gr in res] table.sort() - echo.echo(tabulate.tabulate(table, headers=['PK', 'Label', 'Group type'])) + echo_tabulate(table, headers=['PK', 'Label', 'Group type']) def echo_node_dict(nodes, keys, fmt, identifier, raw, use_attrs=True): diff --git a/aiida/cmdline/commands/cmd_rabbitmq.py b/aiida/cmdline/commands/cmd_rabbitmq.py index 16c9d95dde..7a8d8a7c5f 100644 --- a/aiida/cmdline/commands/cmd_rabbitmq.py +++ b/aiida/cmdline/commands/cmd_rabbitmq.py @@ -16,12 +16,11 @@ import typing as t import click -import tabulate import wrapt from aiida.cmdline.commands.cmd_devel import verdi_devel from aiida.cmdline.params import arguments, options -from aiida.cmdline.utils import decorators, echo +from aiida.cmdline.utils import decorators, echo, echo_tabulate if t.TYPE_CHECKING: import kiwipy.rmq @@ -192,7 +191,7 @@ def cmd_queues_list(client, project, raw, filter_name): headers = [name.capitalize() for name in project] if not raw else [] tablefmt = None if not raw else 'plain' - echo.echo(tabulate.tabulate(output, headers=headers, tablefmt=tablefmt)) + echo_tabulate(output, headers=headers, tablefmt=tablefmt) @cmd_queues.command('create') diff --git a/aiida/cmdline/utils/__init__.py b/aiida/cmdline/utils/__init__.py index a851adef0a..31fe49878c 100644 --- a/aiida/cmdline/utils/__init__.py +++ b/aiida/cmdline/utils/__init__.py @@ -28,6 +28,7 @@ 'echo_info', 'echo_report', 'echo_success', + 'echo_tabulate', 'echo_warning', 'format_call_graph', 'is_verbose', diff --git a/aiida/cmdline/utils/common.py b/aiida/cmdline/utils/common.py index 3073688d53..89f1895f52 100644 --- a/aiida/cmdline/utils/common.py +++ b/aiida/cmdline/utils/common.py @@ -15,7 +15,6 @@ from typing import TYPE_CHECKING from click import style -from tabulate import tabulate from . import echo @@ -25,6 +24,12 @@ __all__ = ('is_verbose',) +def tabulate(table, **kwargs): + """A dummy wrapper to hide the import cost of tabulate""" + import tabulate as tb + return tb.tabulate(table, **kwargs) + + def is_verbose(): """Return whether the configured logging verbosity is considered verbose, i.e., equal or lower to ``INFO`` level. diff --git a/aiida/cmdline/utils/echo.py b/aiida/cmdline/utils/echo.py index 394c5143d6..c5bb0e0b1d 100644 --- a/aiida/cmdline/utils/echo.py +++ b/aiida/cmdline/utils/echo.py @@ -10,7 +10,6 @@ """Convenience functions for logging output from ``verdi`` commands.""" import collections import enum -import json import logging import sys from typing import Any, Optional @@ -19,7 +18,10 @@ CMDLINE_LOGGER = logging.getLogger('verdi') -__all__ = ('echo_report', 'echo_info', 'echo_success', 'echo_warning', 'echo_error', 'echo_critical', 'echo_dictionary') +__all__ = ( + 'echo_report', 'echo_info', 'echo_success', 'echo_warning', 'echo_error', 'echo_critical', 'echo_tabulate', + 'echo_dictionary' +) class ExitCode(enum.IntEnum): @@ -207,6 +209,7 @@ def echo_formatted_list(collection, attributes, sort=None, highlight=None, hide= def _format_dictionary_json_date(dictionary, sort_keys=True): """Return a dictionary formatted as a string using the json format and converting dates to strings.""" + import json def default_jsondump(data): """Function needed to decode datetimes, that would otherwise not be JSON-decodable.""" @@ -241,6 +244,11 @@ def _format_yaml_expanded(dictionary, sort_keys=True): ) +def echo_tabulate(table, **kwargs): + from tabulate import tabulate + echo(tabulate(table, **kwargs)) + + def echo_dictionary(dictionary, fmt='json+date', sort_keys=True): """Log the given dictionary to stdout in the given format From ee62868fab6e8d96ab2f97442e9a8b72371b5075 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 13:29:41 +0100 Subject: [PATCH 13/31] Lazy import inspect, ~7ms Unfortunately, click imports inspect, but that should be fixable upstream. --- aiida/cmdline/commands/cmd_plugin.py | 3 ++- aiida/common/lang.py | 3 ++- aiida/plugins/factories.py | 23 ++++++++++++++++++++++- aiida/plugins/utils.py | 3 ++- aiida/transports/cli.py | 3 ++- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/aiida/cmdline/commands/cmd_plugin.py b/aiida/cmdline/commands/cmd_plugin.py index 2bc4b457fc..68aa5ffce4 100644 --- a/aiida/cmdline/commands/cmd_plugin.py +++ b/aiida/cmdline/commands/cmd_plugin.py @@ -8,7 +8,6 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Command for `verdi plugins`.""" -import inspect import click @@ -27,6 +26,8 @@ def verdi_plugin(): @click.argument('entry_point', type=click.STRING, required=False) def plugin_list(entry_point_group, entry_point): """Display a list of all available plugins.""" + import inspect + from aiida.cmdline.utils.common import print_process_info from aiida.common import EntryPointError from aiida.engine import Process diff --git a/aiida/common/lang.py b/aiida/common/lang.py index d8598eafc3..7206fde896 100644 --- a/aiida/common/lang.py +++ b/aiida/common/lang.py @@ -9,7 +9,6 @@ ########################################################################### """Utilities that extend the basic python language.""" import functools -import inspect import keyword from typing import Any, Callable, Generic, TypeVar @@ -52,6 +51,8 @@ def override_decorator(check=False) -> Callable[[MethodType], MethodType]: """Decorator to signal that a method from a base class is being overridden completely.""" def wrap(func: MethodType) -> MethodType: # pylint: disable=missing-docstring + import inspect + if isinstance(func, property): raise RuntimeError('Override must go after @property decorator') diff --git a/aiida/plugins/factories.py b/aiida/plugins/factories.py index 8b583f1527..0399494998 100644 --- a/aiida/plugins/factories.py +++ b/aiida/plugins/factories.py @@ -11,7 +11,6 @@ """Definition of factories to load classes from the various plugin groups.""" from __future__ import annotations -from inspect import isclass from typing import TYPE_CHECKING, Any, Callable, Literal, NoReturn, Tuple, Type, Union, overload from aiida.common.exceptions import InvalidEntryPointTypeError @@ -84,6 +83,8 @@ def CalculationFactory(entry_point_name: str, load: bool = True) -> Union[EntryP :return: sub class of :py:class:`~aiida.engine.processes.calcjobs.calcjob.CalcJob` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.engine import CalcJob, calcfunction, is_process_function from aiida.orm import CalcFunctionNode @@ -118,6 +119,8 @@ def CalcJobImporterFactory(entry_point_name: str, load: bool = True) -> Union[En :return: the loaded :class:`~aiida.engine.processes.calcjobs.importer.CalcJobImporter` plugin. :raises ``aiida.common.InvalidEntryPointTypeError``: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.engine import CalcJobImporter entry_point_group = 'aiida.calculations.importers' @@ -151,6 +154,8 @@ def DataFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoint, T :return: sub class of :py:class:`~aiida.orm.nodes.data.data.Data` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.orm import Data entry_point_group = 'aiida.data' @@ -184,6 +189,8 @@ def DbImporterFactory(entry_point_name: str, load: bool = True) -> Union[EntryPo :return: sub class of :py:class:`~aiida.tools.dbimporters.baseclasses.DbImporter` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.tools.dbimporters import DbImporter entry_point_group = 'aiida.tools.dbimporters' @@ -217,6 +224,8 @@ def GroupFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoint, :return: sub class of :py:class:`~aiida.orm.groups.Group` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.orm import Group entry_point_group = 'aiida.groups' @@ -250,6 +259,8 @@ def OrbitalFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoint :return: sub class of :py:class:`~aiida.tools.data.orbital.orbital.Orbital` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.tools.data.orbital import Orbital entry_point_group = 'aiida.tools.data.orbitals' @@ -283,6 +294,8 @@ def ParserFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoint, :return: sub class of :py:class:`~aiida.parsers.parser.Parser` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.parsers import Parser entry_point_group = 'aiida.parsers' @@ -316,6 +329,8 @@ def SchedulerFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoi :return: sub class of :py:class:`~aiida.schedulers.scheduler.Scheduler` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.schedulers import Scheduler entry_point_group = 'aiida.schedulers' @@ -349,6 +364,8 @@ def StorageFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoint :return: sub class of :py:class:`~aiida.orm.implementation.storage_backend.StorageBackend`. :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.orm.implementation import StorageBackend entry_point_group = 'aiida.storage' @@ -381,6 +398,8 @@ def TransportFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoi :param load: if True, load the matched entry point and return the loaded resource instead of the entry point itself. :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.transports import Transport entry_point_group = 'aiida.transports' @@ -414,6 +433,8 @@ def WorkflowFactory(entry_point_name: str, load: bool = True) -> Union[EntryPoin :return: sub class of :py:class:`~aiida.engine.processes.workchains.workchain.WorkChain` or a `workfunction` :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. """ + from inspect import isclass + from aiida.engine import WorkChain, is_process_function, workfunction from aiida.orm import WorkFunctionNode diff --git a/aiida/plugins/utils.py b/aiida/plugins/utils.py index 2d34ffdb6b..b90be8ffaf 100644 --- a/aiida/plugins/utils.py +++ b/aiida/plugins/utils.py @@ -11,7 +11,6 @@ from __future__ import annotations from importlib import import_module -from inspect import isclass, isfunction from logging import Logger from types import FunctionType import typing as t @@ -55,6 +54,8 @@ def get_version_info(self, plugin: str | type) -> dict[t.Any, dict[t.Any, t.Any] :raises TypeError: If ``plugin`` (or the resource pointed to it in the case of an entry point) is not a class or a function. """ + from inspect import isclass, isfunction + from aiida import __version__ as version_core if isinstance(plugin, str): diff --git a/aiida/transports/cli.py b/aiida/transports/cli.py index 8ea8901e37..037c0391d9 100644 --- a/aiida/transports/cli.py +++ b/aiida/transports/cli.py @@ -9,7 +9,6 @@ ########################################################################### """Common cli utilities for transport plugins.""" from functools import partial -import inspect import click @@ -57,6 +56,8 @@ def common_params(command_func): def transport_option_default(name, computer): """Determine the default value for an auth_param key.""" + import inspect + transport_cls = computer.get_transport_class() suggester_name = f'_get_{name}_suggestion_string' members = dict(inspect.getmembers(transport_cls)) From 5c4488d894d8e49a266bc44b4a4e4d7fe2404694 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 13:34:59 +0100 Subject: [PATCH 14/31] Lazy imports of stdlib, ~15ms hashlib, socket, pytz, shutil, tempfile, importlib.resources --- aiida/cmdline/params/options/commands/setup.py | 5 ++++- aiida/cmdline/params/types/path.py | 7 ++++--- aiida/common/timezone.py | 4 ++-- aiida/manage/configuration/__init__.py | 10 +++++++--- aiida/manage/configuration/config.py | 13 ++++++++++--- aiida/manage/configuration/settings.py | 2 +- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/aiida/cmdline/params/options/commands/setup.py b/aiida/cmdline/params/options/commands/setup.py index b3218bdda5..88d7216072 100644 --- a/aiida/cmdline/params/options/commands/setup.py +++ b/aiida/cmdline/params/options/commands/setup.py @@ -10,7 +10,6 @@ """Reusable command line interface options for the setup commands.""" import functools import getpass -import hashlib import click @@ -94,6 +93,8 @@ def get_quicksetup_database_name(ctx, param, value): # pylint: disable=unused-a :param ctx: click context which should contain the contextual parameters :return: the database name """ + import hashlib + if value is not None: return value @@ -114,6 +115,8 @@ def get_quicksetup_username(ctx, param, value): # pylint: disable=unused-argume :param ctx: click context which should contain the contextual parameters :return: the username """ + import hashlib + if value is not None: return value diff --git a/aiida/cmdline/params/types/path.py b/aiida/cmdline/params/types/path.py index 33de95513c..2d1e03196b 100644 --- a/aiida/cmdline/params/types/path.py +++ b/aiida/cmdline/params/types/path.py @@ -9,7 +9,6 @@ ########################################################################### """Click parameter types for paths.""" import os -from socket import timeout import click @@ -86,13 +85,14 @@ def convert(self, value, param, ctx): def checks_url(self, url, param, ctx): """Check whether URL is reachable within timeout.""" + import socket import urllib.error import urllib.request try: with urllib.request.urlopen(url, timeout=self.timeout_seconds): pass - except (urllib.error.URLError, urllib.error.HTTPError, timeout): + except (urllib.error.URLError, urllib.error.HTTPError, socket.timeout): self.fail(f'{self.name} "{url}" could not be reached within {self.timeout_seconds} s.\n', param, ctx) return url @@ -124,10 +124,11 @@ def convert(self, value, param, ctx): def get_url(self, url, param, ctx): """Retrieve file from URL.""" + import socket import urllib.error import urllib.request try: return urllib.request.urlopen(url, timeout=self.timeout_seconds) # pylint: disable=consider-using-with - except (urllib.error.URLError, urllib.error.HTTPError, timeout): + except (urllib.error.URLError, urllib.error.HTTPError, socket.timeout): self.fail(f'{self.name} "{url}" could not be reached within {self.timeout_seconds} s.\n', param, ctx) diff --git a/aiida/common/timezone.py b/aiida/common/timezone.py index 60e3db7714..0419fd22e6 100644 --- a/aiida/common/timezone.py +++ b/aiida/common/timezone.py @@ -11,8 +11,6 @@ from datetime import datetime, timedelta, timezone, tzinfo from typing import Optional -import pytz - utc = timezone.utc # Simply forward this attribute from the :mod:`datetime.timezone` built in library @@ -51,6 +49,8 @@ def timezone_from_name(name: str) -> tzinfo: :returns: The corresponding timezone object. :raises ValueError: if the timezone name is unknown. """ + import pytz + try: return pytz.timezone(name) except pytz.exceptions.UnknownTimeZoneError as exception: diff --git a/aiida/manage/configuration/__init__.py b/aiida/manage/configuration/__init__.py index 11ca28d3a0..ad43c056f8 100644 --- a/aiida/manage/configuration/__init__.py +++ b/aiida/manage/configuration/__init__.py @@ -50,7 +50,6 @@ from contextlib import contextmanager import os -import shutil from typing import TYPE_CHECKING, Any, Optional import warnings @@ -104,11 +103,17 @@ def load_config(create=False) -> 'Config': def _merge_deprecated_cache_yaml(config, filepath): """Merge the deprecated cache_config.yml into the config.""" - from aiida.common import timezone cache_path = os.path.join(os.path.dirname(filepath), 'cache_config.yml') if not os.path.exists(cache_path): return + # Imports are here to avoid them when not needed + import shutil + + import yaml + + from aiida.common import timezone + cache_path_backup = None # Keep generating a new backup filename based on the current time until it does not exist while not cache_path_backup or os.path.isfile(cache_path_backup): @@ -119,7 +124,6 @@ def _merge_deprecated_cache_yaml(config, filepath): f'moving to: {cache_path_backup}', version=3 ) - import yaml with open(cache_path, 'r', encoding='utf8') as handle: cache_config = yaml.safe_load(handle) for profile_name, data in cache_config.items(): diff --git a/aiida/manage/configuration/config.py b/aiida/manage/configuration/config.py index 9db56fc43e..c50439caa9 100644 --- a/aiida/manage/configuration/config.py +++ b/aiida/manage/configuration/config.py @@ -10,11 +10,8 @@ """Module that defines the configuration file of an AiiDA instance and functions to create and load it.""" import codecs from functools import cache -from importlib.resources import files import json import os -import shutil -import tempfile from typing import Any, Dict, Optional, Sequence, Tuple from aiida.common.exceptions import ConfigurationError @@ -31,6 +28,8 @@ @cache def config_schema() -> Dict[str, Any]: """Return the configuration schema.""" + from importlib.resources import files + return json.loads(files(schema_module).joinpath(SCHEMA_FILE).read_text(encoding='utf8')) @@ -109,6 +108,8 @@ def _backup(cls, filepath): :param filepath: absolute path to the configuration file to backup :return: the absolute path of the created backup """ + import shutil + from aiida.common import timezone filepath_backup = None @@ -345,6 +346,8 @@ def delete_profile( :param include_database_user: also delete the database user configured for the profile. :param include_repository: also delete the repository configured for the profile. """ + import shutil + from aiida.manage.external.postgres import Postgres profile = self.get_profile(name) @@ -483,6 +486,8 @@ def store(self): :return: self """ + import tempfile + from aiida.common.files import md5_file, md5_from_filelike from .settings import DEFAULT_CONFIG_INDENT_SIZE @@ -515,6 +520,8 @@ def _atomic_write(self, filepath=None): :param filepath: optional filepath to write the contents to, if not specified, the default filename is used. """ + import tempfile + from .settings import DEFAULT_CONFIG_INDENT_SIZE, DEFAULT_UMASK umask = os.umask(DEFAULT_UMASK) diff --git a/aiida/manage/configuration/settings.py b/aiida/manage/configuration/settings.py index 1236055468..71c2576ba9 100644 --- a/aiida/manage/configuration/settings.py +++ b/aiida/manage/configuration/settings.py @@ -38,7 +38,7 @@ def create_instance_directories(): """ from aiida.common import ConfigurationError - directory_base = pathlib.Path(AIIDA_CONFIG_FOLDER).expanduser() + directory_base = AIIDA_CONFIG_FOLDER.expanduser() directory_daemon = directory_base / DAEMON_DIR directory_daemon_log = directory_base / DAEMON_LOG_DIR directory_access = directory_base / ACCESS_CONTROL_DIR From fd15352669326cda6672387e41712a5b52a017ec Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 13:48:06 +0100 Subject: [PATCH 15/31] One more temporary fix for tests --- aiida/cmdline/params/types/plugin.py | 2 +- aiida/cmdline/utils/echo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index f21683e6e5..47a7f54baf 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -68,7 +68,7 @@ def __init__(self, group=None, load=False, *args, **kwargs): """ # pylint: disable=keyword-arg-before-vararg self.load = load - self._groups = None + self._groups = [] self._entry_points = [] self._entry_point_names = [] # TODO: Implement the following logic lazily diff --git a/aiida/cmdline/utils/echo.py b/aiida/cmdline/utils/echo.py index c5bb0e0b1d..6c9bd5a882 100644 --- a/aiida/cmdline/utils/echo.py +++ b/aiida/cmdline/utils/echo.py @@ -10,6 +10,7 @@ """Convenience functions for logging output from ``verdi`` commands.""" import collections import enum +import json import logging import sys from typing import Any, Optional @@ -209,7 +210,6 @@ def echo_formatted_list(collection, attributes, sort=None, highlight=None, hide= def _format_dictionary_json_date(dictionary, sort_keys=True): """Return a dictionary formatted as a string using the json format and converting dates to strings.""" - import json def default_jsondump(data): """Function needed to decode datetimes, that would otherwise not be JSON-decodable.""" From 57e1f26cecc928dd87ec7fc2e132c1b97f6b0554 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 14:48:55 +0100 Subject: [PATCH 16/31] Lazy PluginParamType --- aiida/cmdline/params/types/plugin.py | 70 +++++++++++++--------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 47a7f54baf..073650246a 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -68,57 +68,51 @@ def __init__(self, group=None, load=False, *args, **kwargs): """ # pylint: disable=keyword-arg-before-vararg self.load = load - self._groups = [] - self._entry_points = [] - self._entry_point_names = [] - # TODO: Implement the following logic lazily - return + self._input_group = group + + super().__init__(*args, **kwargs) + + def _get_valid_groups(self): + """Get allowed groups for this instance""" + + group = self._input_group valid_entry_point_groups = get_entry_point_groups() if group is None: - self._groups = tuple(valid_entry_point_groups) + return tuple(valid_entry_point_groups) + + if isinstance(group, str): + unvalidated_groups = (group,) + elif isinstance(group, tuple): + unvalidated_groups = group else: - if isinstance(group, str): - invalidated_groups = (group,) - elif isinstance(group, tuple): - invalidated_groups = group - else: - raise ValueError('invalid type for group') + raise ValueError('invalid type for group') - groups = [] + groups = [] - for grp in invalidated_groups: + for grp in unvalidated_groups: - if not grp.startswith(ENTRY_POINT_GROUP_PREFIX): - grp = ENTRY_POINT_GROUP_PREFIX + grp + if not grp.startswith(ENTRY_POINT_GROUP_PREFIX): + grp = ENTRY_POINT_GROUP_PREFIX + grp - if grp not in valid_entry_point_groups: - raise ValueError(f'entry point group {grp} is not recognized') + if grp not in valid_entry_point_groups: + raise ValueError(f'entry point group {grp} is not recognized') - groups.append(grp) + groups.append(grp) - self._groups = tuple(groups) + return tuple(groups) - self._init_entry_points() - self.load = load + @functools.cached_property + def groups(self) -> tuple: + return self._get_valid_groups() - super().__init__(*args, **kwargs) + @functools.cached_property + def _entry_points(self) -> list[tuple]: + return [(group, entry_point) for group in self.groups for entry_point in get_entry_points(group)] - def _init_entry_points(self): - """ - Populate entry point information that will be used later on. This should only be called - once in the constructor after setting self.groups because the groups should not be changed - after instantiation - """ - # TODO: Optimize this - self._entry_points = [(group, entry_point) for group in self.groups for entry_point in get_entry_points(group)] - self._entry_point_names = [entry_point.name for group in self.groups for entry_point in get_entry_points(group)] - - @property - def groups(self): - # TODO: Remove this - # raise ValueError("Whoops groups") - return self._groups + @functools.cached_property + def _entry_point_names(self) -> list[str]: + return [entry_point.name for group in self.groups for entry_point in get_entry_points(group)] @property def has_potential_ambiguity(self): From 4174c80d1aa28fd499c916883c6fb834d9145a74 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 15:22:23 +0100 Subject: [PATCH 17/31] lazy IdentifierParamType --- aiida/cmdline/params/types/identifier.py | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index 734c4da289..a7bad12d5b 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -13,6 +13,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from functools import cached_property import click @@ -48,26 +49,29 @@ def __init__(self, sub_classes: tuple[str] | None = None): will be mapped upon. These classes have to be strict sub classes of the base orm class defined by the orm class loader """ - from aiida.common import exceptions + if sub_classes is not None and not isinstance(sub_classes, tuple): + raise TypeError('sub_classes should be a tuple of entry point strings') self._sub_classes: tuple | None = None - self._entry_points = [] + self._entry_point_strings = sub_classes - if sub_classes is None: - return + @cached_property + def _entry_points(self): + """Allowed entry points, loaded on demand""" + from aiida.common import exceptions - if not isinstance(sub_classes, tuple): - raise TypeError('sub_classes should be a tuple of entry point strings') + if self._entry_point_strings is None: + return None - # TODO: Add a property that loads all this on demand - return - for entry_point_string in sub_classes: + entry_points = [] + for entry_point_string in self._entry_point_strings: try: entry_point = get_entry_point_from_string(entry_point_string) except (ValueError, exceptions.EntryPointError) as exception: raise ValueError(f'{entry_point_string} is not a valid entry point string: {exception}') else: - self._entry_points.append(entry_point) + entry_points.append(entry_point) + return entry_points @property @abstractmethod @@ -93,9 +97,6 @@ def convert(self, value, param, ctx): from aiida.common import exceptions from aiida.orm.utils.loaders import OrmEntityLoader - # TODO: Remove this - # raise ValueError("") - value = super().convert(value, param, ctx) if not value: @@ -106,7 +107,7 @@ def convert(self, value, param, ctx): if not issubclass(loader, OrmEntityLoader): raise RuntimeError('the orm class loader should be a subclass of OrmEntityLoader') - # If entry points where in the constructor, we load their corresponding classes, validate that they are valid + # If entry points were in the constructor, we load their corresponding classes, validate that they are valid # sub classes of the orm class loader and then pass it as the sub_class parameter to the load_entity call. # We store the loaded entry points in an instance variable, such that the loading only has to be done once. if self._entry_points and self._sub_classes is None: From 615a0aa993ec1261c5425ae9cc142a2917e65863 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 18 Oct 2023 16:06:17 +0100 Subject: [PATCH 18/31] Update tests --- aiida/cmdline/params/types/identifier.py | 8 +++----- tests/cmdline/params/types/test_identifier.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index a7bad12d5b..9c650fe578 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -10,8 +10,6 @@ """ Module for custom click param type identifier """ -from __future__ import annotations - from abc import ABC, abstractmethod from functools import cached_property @@ -33,7 +31,7 @@ class IdentifierParamType(click.ParamType, ABC): which should be a subclass of `aiida.orm.utils.loaders.OrmEntityLoader` for the corresponding orm class. """ - def __init__(self, sub_classes: tuple[str] | None = None): + def __init__(self, sub_classes=None): """ Construct the parameter type, optionally specifying a tuple of entry points that reference classes that should be a sub class of the base orm class of the orm class loader. The classes pointed to by @@ -52,7 +50,7 @@ def __init__(self, sub_classes: tuple[str] | None = None): if sub_classes is not None and not isinstance(sub_classes, tuple): raise TypeError('sub_classes should be a tuple of entry point strings') - self._sub_classes: tuple | None = None + self._sub_classes = None self._entry_point_strings = sub_classes @cached_property @@ -61,7 +59,7 @@ def _entry_points(self): from aiida.common import exceptions if self._entry_point_strings is None: - return None + return [] entry_points = [] for entry_point_string in self._entry_point_strings: diff --git a/tests/cmdline/params/types/test_identifier.py b/tests/cmdline/params/types/test_identifier.py index 00c98e6dd5..09cd74e03b 100644 --- a/tests/cmdline/params/types/test_identifier.py +++ b/tests/cmdline/params/types/test_identifier.py @@ -32,18 +32,25 @@ def test_identifier_sub_invalid_type(self): with pytest.raises(TypeError): NodeParamType(sub_classes='aiida.data:core.structure') + # NOTE: We load and validate entry points lazily so we need + # to access them to raise. with pytest.raises(TypeError): - NodeParamType(sub_classes=(None,)) + npt = NodeParamType(sub_classes=(None,)) + npt._entry_points # pylint: disable=protected-access,pointless-statement def test_identifier_sub_invalid_entry_point(self): """ The sub_classes keyword argument should expect a tuple of valid entry point strings """ + # NOTE: We load and validate entry points lazily so we need + # to access them to raise. with pytest.raises(ValueError): - NodeParamType(sub_classes=('aiida.data.structure',)) + npt = NodeParamType(sub_classes=('aiida.data.structure',)) + npt._entry_points # pylint: disable=protected-access,pointless-statement with pytest.raises(ValueError): NodeParamType(sub_classes=('aiida.data:not_existent',)) + npt._entry_points # pylint: disable=protected-access,pointless-statement def test_identifier_sub_classes(self): """ From 82b950afd9b65e9ca90a3ee2158adaa0a28e2bcf Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 15:29:27 +0100 Subject: [PATCH 19/31] Update aiida/cmdline/utils/echo.py Co-authored-by: Sebastiaan Huber --- aiida/cmdline/utils/echo.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aiida/cmdline/utils/echo.py b/aiida/cmdline/utils/echo.py index 6c9bd5a882..27f2debff3 100644 --- a/aiida/cmdline/utils/echo.py +++ b/aiida/cmdline/utils/echo.py @@ -245,6 +245,14 @@ def _format_yaml_expanded(dictionary, sort_keys=True): def echo_tabulate(table, **kwargs): + """Echo the string generated by passing ``table`` to ``tabulate.tabulate``. + + This wrapper is added in order to lazily import the ``tabulate`` package only when invoked. This helps keeping the + import time of the :mod:`aiida.cmdline` to a minimum, which is critical for keeping tab-completion snappy. + + :param table: The table of data to echo. + :param kwargs: Additional arguments passed to :meth:`tabulate.tabulate`. + """ from tabulate import tabulate echo(tabulate(table, **kwargs)) From ceb3ec9f79d52efd4ca6f3bf4d926ed966c274ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:30:39 +0000 Subject: [PATCH 20/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiida/cmdline/utils/echo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiida/cmdline/utils/echo.py b/aiida/cmdline/utils/echo.py index 27f2debff3..5cef7732dc 100644 --- a/aiida/cmdline/utils/echo.py +++ b/aiida/cmdline/utils/echo.py @@ -246,7 +246,7 @@ def _format_yaml_expanded(dictionary, sort_keys=True): def echo_tabulate(table, **kwargs): """Echo the string generated by passing ``table`` to ``tabulate.tabulate``. - + This wrapper is added in order to lazily import the ``tabulate`` package only when invoked. This helps keeping the import time of the :mod:`aiida.cmdline` to a minimum, which is critical for keeping tab-completion snappy. From 8d88334c7b2e3bf5922b1681a58d36574ac60320 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 16:34:14 +0100 Subject: [PATCH 21/31] Minor refactoring and typing fix in configuration.py --- aiida/manage/configuration/settings.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/aiida/manage/configuration/settings.py b/aiida/manage/configuration/settings.py index 71c2576ba9..9c75d284eb 100644 --- a/aiida/manage/configuration/settings.py +++ b/aiida/manage/configuration/settings.py @@ -10,7 +10,7 @@ """Base settings required for the configuration of an AiiDA instance.""" import os import pathlib -import typing +import typing as t import warnings DEFAULT_UMASK = 0o0077 @@ -24,13 +24,13 @@ DEFAULT_DAEMON_LOG_DIR_NAME = 'log' DEFAULT_ACCESS_CONTROL_DIR_NAME = 'access' -AIIDA_CONFIG_FOLDER: typing.Optional[pathlib.Path] = None -DAEMON_DIR: typing.Optional[pathlib.Path] = None -DAEMON_LOG_DIR: typing.Optional[pathlib.Path] = None -ACCESS_CONTROL_DIR: typing.Optional[pathlib.Path] = None +AIIDA_CONFIG_FOLDER: t.Optional[pathlib.Path] = None +DAEMON_DIR: t.Optional[pathlib.Path] = None +DAEMON_LOG_DIR: t.Optional[pathlib.Path] = None +ACCESS_CONTROL_DIR: t.Optional[pathlib.Path] = None -def create_instance_directories(): +def create_instance_directories() -> None: """Create the base directories required for a new AiiDA instance. This will create the base AiiDA directory defined by the AIIDA_CONFIG_FOLDER variable, unless it already exists. @@ -66,7 +66,7 @@ def create_instance_directories(): os.umask(umask) -def set_configuration_directory(aiida_config_folder: pathlib.Path = None): +def set_configuration_directory(aiida_config_folder: t.Optional[pathlib.Path] = None) -> None: """Determine location of configuration directory, set related global variables and create instance directories. The location of the configuration folder will be determined and optionally created following these heuristics: @@ -86,11 +86,11 @@ def set_configuration_directory(aiida_config_folder: pathlib.Path = None): global DAEMON_LOG_DIR global ACCESS_CONTROL_DIR - environment_variable = os.environ.get(DEFAULT_AIIDA_PATH_VARIABLE, None) - if aiida_config_folder is not None: + AIIDA_CONFIG_FOLDER = aiida_config_folder - elif environment_variable: + + elif environment_variable := os.environ.get(DEFAULT_AIIDA_PATH_VARIABLE): # Loop over all the paths in the `AIIDA_PATH` variable to see if any of them contain a configuration folder for base_dir_path in [path for path in environment_variable.split(':') if path]: @@ -107,7 +107,7 @@ def set_configuration_directory(aiida_config_folder: pathlib.Path = None): break else: - # The `AIIDA_PATH` variable is not set, so default to the default path and try to create it if it does not exist + # The `AIIDA_PATH` variable is not set so use the default path and try to create it if it does not exist AIIDA_CONFIG_FOLDER = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME DAEMON_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME From 7a2cc4408310ba21a535d0a11eefa3276fd3b5d8 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 16:36:11 +0100 Subject: [PATCH 22/31] review cleanup --- aiida/plugins/entry_point.py | 4 ---- tests/cmdline/params/types/test_identifier.py | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/aiida/plugins/entry_point.py b/aiida/plugins/entry_point.py index 60328abdcf..a4d7775079 100644 --- a/aiida/plugins/entry_point.py +++ b/aiida/plugins/entry_point.py @@ -43,7 +43,6 @@ def eps() -> EntryPoints: which will always iterate over all entry points since it looks for possible duplicate entries. """ - # raise ValueError("eps!") from importlib_metadata import EntryPoints, entry_points all_eps = entry_points() return EntryPoints(sorted(all_eps, key=lambda x: x.group)) @@ -55,9 +54,6 @@ def eps_select(group: str, name: str | None = None) -> EntryPoints: A thin wrapper around entry_points.select() calls, which are expensive so we want to cache them. """ - # msg = f"{group=} {name=}" - # print(msg) - # raise ValueError(msg) if name is None: return eps().select(group=group) return eps().select(group=group, name=name) diff --git a/tests/cmdline/params/types/test_identifier.py b/tests/cmdline/params/types/test_identifier.py index 09cd74e03b..8de8383f99 100644 --- a/tests/cmdline/params/types/test_identifier.py +++ b/tests/cmdline/params/types/test_identifier.py @@ -32,8 +32,7 @@ def test_identifier_sub_invalid_type(self): with pytest.raises(TypeError): NodeParamType(sub_classes='aiida.data:core.structure') - # NOTE: We load and validate entry points lazily so we need - # to access them to raise. + # NOTE: We load and validate entry points lazily so we need to access them to raise. with pytest.raises(TypeError): npt = NodeParamType(sub_classes=(None,)) npt._entry_points # pylint: disable=protected-access,pointless-statement @@ -42,8 +41,7 @@ def test_identifier_sub_invalid_entry_point(self): """ The sub_classes keyword argument should expect a tuple of valid entry point strings """ - # NOTE: We load and validate entry points lazily so we need - # to access them to raise. + # NOTE: We load and validate entry points lazily so we need to access them to raise. with pytest.raises(ValueError): npt = NodeParamType(sub_classes=('aiida.data.structure',)) npt._entry_points # pylint: disable=protected-access,pointless-statement From 2b82927f598c72dff7b94ae7b378980af15534d5 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 16:49:35 +0100 Subject: [PATCH 23/31] Update docstring --- aiida/cmdline/params/types/identifier.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index 9c650fe578..f138cb30fb 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -40,8 +40,7 @@ def __init__(self, sub_classes=None): To prevent having to load the database environment at import time, the actual loading of the entry points is deferred until the call to `convert` is made. This is to keep the command line autocompletion light - and responsive. The entry point strings will be validated, however, to see if the correspond to known - entry points. + and responsive. We also postpone the validation of entry point strings for the same reason. :param sub_classes: a tuple of entry point strings that can narrow the set of orm classes that values will be mapped upon. These classes have to be strict sub classes of the base orm class defined From d4b24914dee90475beb6d80d56845de97ffe22e9 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 17:26:13 +0100 Subject: [PATCH 24/31] Optimization and typing in params/types/plugin.py --- .pre-commit-config.yaml | 2 -- aiida/cmdline/groups/dynamic.py | 7 +++-- aiida/cmdline/params/types/plugin.py | 40 +++++++++++++++----------- docs/source/reference/command_line.rst | 15 ---------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54074affef..326ec34429 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,13 +93,11 @@ repos: aiida/cmdline/commands/cmd_node.py| aiida/cmdline/commands/cmd_shell.py| aiida/cmdline/commands/cmd_storage.py| - aiida/cmdline/groups/dynamic.py| aiida/cmdline/params/options/commands/setup.py| aiida/cmdline/params/options/interactive.py| aiida/cmdline/params/options/main.py| aiida/cmdline/params/options/multivalue.py| aiida/cmdline/params/types/group.py| - aiida/cmdline/params/types/plugin.py| aiida/cmdline/utils/ascii_vis.py| aiida/cmdline/utils/common.py| aiida/cmdline/utils/echo.py| diff --git a/aiida/cmdline/groups/dynamic.py b/aiida/cmdline/groups/dynamic.py index caead9d3ae..6503b031ba 100644 --- a/aiida/cmdline/groups/dynamic.py +++ b/aiida/cmdline/groups/dynamic.py @@ -5,6 +5,7 @@ import copy import functools import re +import typing as t import click @@ -56,7 +57,7 @@ def __init__( self.factory = ENTRY_POINT_GROUP_FACTORY_MAPPING[entry_point_group] self.shared_options = shared_options - def list_commands(self, ctx) -> list[str]: + def list_commands(self, ctx: click.Context) -> list[str]: """Return the sorted list of subcommands for this group. :param ctx: The :class:`click.Context`. @@ -68,7 +69,7 @@ def list_commands(self, ctx) -> list[str]: ]) return sorted(commands) - def get_command(self, ctx, cmd_name): + def get_command(self, ctx: click.Context, cmd_name) -> t.Any: """Return the command with the given name. :param ctx: The :class:`click.Context`. @@ -81,7 +82,7 @@ def get_command(self, ctx, cmd_name): command = super().get_command(ctx, cmd_name) return command - def create_command(self, ctx, entry_point): + def create_command(self, ctx: click.Context, entry_point: str) -> t.Any: """Create a subcommand for the given ``entry_point``.""" cls = self.factory(entry_point) command = functools.partial(self.command, ctx, cls) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 073650246a..8a7ed0c557 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -8,7 +8,10 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Click parameter type for AiiDA Plugins.""" +from __future__ import annotations + import functools +import typing as t import click @@ -27,6 +30,9 @@ from .strings import EntryPointType +if t.TYPE_CHECKING: + from importlib_metadata import EntryPoint + __all__ = ('PluginParamType',) @@ -61,10 +67,10 @@ class PluginParamType(EntryPointType): 'aiida.workflows': factories.WorkflowFactory, } - def __init__(self, group=None, load=False, *args, **kwargs): + def __init__(self, group: str | tuple[str] | None = None, load: bool = False, *args, **kwargs): """ - Validate that group is either a string or a tuple of valid entry point groups, or if it - is not specified use the tuple of all recognized entry point groups. + group should be either a string or a tuple of valid entry point groups. + If it is not specified we use the tuple of all recognized entry point groups. """ # pylint: disable=keyword-arg-before-vararg self.load = load @@ -72,7 +78,7 @@ def __init__(self, group=None, load=False, *args, **kwargs): super().__init__(*args, **kwargs) - def _get_valid_groups(self): + def _get_valid_groups(self) -> tuple[str]: """Get allowed groups for this instance""" group = self._input_group @@ -103,28 +109,28 @@ def _get_valid_groups(self): return tuple(groups) @functools.cached_property - def groups(self) -> tuple: + def groups(self) -> tuple[str]: return self._get_valid_groups() @functools.cached_property - def _entry_points(self) -> list[tuple]: + def _entry_points(self) -> list[tuple[str, EntryPoint]]: return [(group, entry_point) for group in self.groups for entry_point in get_entry_points(group)] @functools.cached_property def _entry_point_names(self) -> list[str]: - return [entry_point.name for group in self.groups for entry_point in get_entry_points(group)] + return [entry_point.name for _, entry_point in self._entry_points] @property - def has_potential_ambiguity(self): + def has_potential_ambiguity(self) -> bool: """ Returns whether the set of supported entry point groups can lead to ambiguity when only an entry point name is specified. This will happen if one ore more groups share an entry point with a common name """ return len(self._entry_point_names) != len(set(self._entry_point_names)) - def get_valid_arguments(self): + def get_valid_arguments(self) -> list[str]: """ - Return a list of all available plugins for the groups configured for this PluginParamType instance. + Return a list of all available plugin names for the groups configured for this PluginParamType instance. If the entry point names are not unique, because there are multiple groups that contain an entry point that has an identical name, we need to prefix the names with the full group name @@ -136,7 +142,7 @@ def get_valid_arguments(self): return sorted(self._entry_point_names) - def get_possibilities(self, incomplete=''): + def get_possibilities(self, incomplete: str = '') -> list[str]: """ Return a list of plugins starting with incomplete """ @@ -161,7 +167,9 @@ def get_possibilities(self, incomplete=''): return possibilites - def shell_complete(self, ctx, param, incomplete): # pylint: disable=unused-argument + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> list[click.shell_completion.CompletionItem]: # pylint: disable=unused-argument """ Return possible completions based on an incomplete value @@ -169,10 +177,10 @@ def shell_complete(self, ctx, param, incomplete): # pylint: disable=unused-argu """ return [click.shell_completion.CompletionItem(p) for p in self.get_possibilities(incomplete=incomplete)] - def get_missing_message(self, param): # pylint: disable=unused-argument + def get_missing_message(self, param: click.Parameter) -> str: # pylint: disable=unused-argument return 'Possible arguments are:\n\n' + '\n'.join(self.get_valid_arguments()) - def get_entry_point_from_string(self, entry_point_string): + def get_entry_point_from_string(self, entry_point_string: str) -> EntryPoint: """ Validate a given entry point string, which means that it should have a valid entry point string format and that the entry point unambiguously corresponds to an entry point in the groups configured for this @@ -229,11 +237,11 @@ def get_entry_point_from_string(self, entry_point_string): except exceptions.EntryPointError as exception: raise ValueError(exception) - def validate_entry_point_group(self, group): + def validate_entry_point_group(self, group: str) -> None: if group not in self.groups: raise ValueError(f'entry point group `{group}` is not supported by this parameter.') - def convert(self, value, param, ctx): + def convert(self, value: t.Any, param: click.Parameter, ctx: click.Context) -> t.Any: """ Convert the string value to an entry point instance, if the value can be successfully parsed into an actual entry point. Will raise click.BadParameter if validation fails. diff --git a/docs/source/reference/command_line.rst b/docs/source/reference/command_line.rst index c3ae6e2f7c..fcc0f4e17f 100644 --- a/docs/source/reference/command_line.rst +++ b/docs/source/reference/command_line.rst @@ -571,21 +571,6 @@ Below is a list with all available subcommands. version Print the current version of the storage schema. -.. _reference:command-line:verdi-tui: - -``verdi tui`` -------------- - -.. code:: console - - Usage: [OPTIONS] - - Open Textual TUI. - - Options: - --help Show this message and exit. - - .. _reference:command-line:verdi-user: ``verdi user`` From 3e945c01739e341f43fa1570f937231d61c8216f Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 19:41:14 +0100 Subject: [PATCH 25/31] Enable typing for params/types/plugin.py|identifier.py and aiida/manage/configuration/settings.py --- .pre-commit-config.yaml | 2 +- aiida/cmdline/params/types/identifier.py | 22 +++++++++++++++------- aiida/cmdline/params/types/plugin.py | 19 ++++++------------- aiida/manage/configuration/settings.py | 18 ++++++++---------- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 326ec34429..8ea8a03175 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,6 +93,7 @@ repos: aiida/cmdline/commands/cmd_node.py| aiida/cmdline/commands/cmd_shell.py| aiida/cmdline/commands/cmd_storage.py| + aiida/cmdline/groups/dynamic.py| aiida/cmdline/params/options/commands/setup.py| aiida/cmdline/params/options/interactive.py| aiida/cmdline/params/options/main.py| @@ -113,7 +114,6 @@ repos: aiida/manage/configuration/__init__.py| aiida/manage/configuration/config.py| aiida/manage/configuration/profile.py| - aiida/manage/configuration/settings.py| aiida/manage/external/rmq/launcher.py| aiida/manage/tests/main.py| aiida/manage/tests/pytest_fixtures.py| diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index f138cb30fb..4d521661f7 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -10,14 +10,22 @@ """ Module for custom click param type identifier """ +from __future__ import annotations + from abc import ABC, abstractmethod from functools import cached_property +import typing as t import click from aiida.cmdline.utils.decorators import with_dbenv from aiida.plugins.entry_point import get_entry_point_from_string +if t.TYPE_CHECKING: + from importlib_metadata import EntryPoint + + from aiida.orm.utils.loaders import OrmEntityLoader + __all__ = ('IdentifierParamType',) @@ -31,7 +39,7 @@ class IdentifierParamType(click.ParamType, ABC): which should be a subclass of `aiida.orm.utils.loaders.OrmEntityLoader` for the corresponding orm class. """ - def __init__(self, sub_classes=None): + def __init__(self, sub_classes: tuple[str, ...] | None = None): """ Construct the parameter type, optionally specifying a tuple of entry points that reference classes that should be a sub class of the base orm class of the orm class loader. The classes pointed to by @@ -49,11 +57,11 @@ def __init__(self, sub_classes=None): if sub_classes is not None and not isinstance(sub_classes, tuple): raise TypeError('sub_classes should be a tuple of entry point strings') - self._sub_classes = None + self._sub_classes: tuple | None = None self._entry_point_strings = sub_classes @cached_property - def _entry_points(self): + def _entry_points(self) -> list[EntryPoint]: """Allowed entry points, loaded on demand""" from aiida.common import exceptions @@ -72,8 +80,8 @@ def _entry_points(self): @property @abstractmethod - @with_dbenv() - def orm_class_loader(self): + @with_dbenv() # type: ignore[misc] + def orm_class_loader(self) -> OrmEntityLoader: """ Return the orm entity loader class, which should be a subclass of OrmEntityLoader. This class is supposed to be used to load the entity for a given identifier @@ -81,8 +89,8 @@ def orm_class_loader(self): :return: the orm entity loader class for this ParamType """ - @with_dbenv() - def convert(self, value, param, ctx): + @with_dbenv() # type: ignore[misc] + def convert(self, value: t.Any, param: click.Parameter | None, ctx: click.Context) -> t.Any: """ Attempt to convert the given value to an instance of the orm class using the orm class loader. diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index 8a7ed0c557..ca919e1b90 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -78,7 +78,7 @@ def __init__(self, group: str | tuple[str] | None = None, load: bool = False, *a super().__init__(*args, **kwargs) - def _get_valid_groups(self) -> tuple[str]: + def _get_valid_groups(self) -> tuple[str, ...]: """Get allowed groups for this instance""" group = self._input_group @@ -109,7 +109,7 @@ def _get_valid_groups(self) -> tuple[str]: return tuple(groups) @functools.cached_property - def groups(self) -> tuple[str]: + def groups(self) -> tuple[str, ...]: return self._get_valid_groups() @functools.cached_property @@ -168,7 +168,7 @@ def get_possibilities(self, incomplete: str = '') -> list[str]: return possibilites def shell_complete( - self, ctx: click.Context, param: click.Parameter, incomplete: str + self, ctx: click.Context | None, param: click.Parameter | None, incomplete: str ) -> list[click.shell_completion.CompletionItem]: # pylint: disable=unused-argument """ Return possible completions based on an incomplete value @@ -189,8 +189,6 @@ def get_entry_point_from_string(self, entry_point_string: str) -> EntryPoint: :returns: the entry point if valid :raises: ValueError if the entry point string is invalid """ - group = None - name = None entry_point_format = get_entry_point_string_format(entry_point_string) @@ -226,14 +224,8 @@ def get_entry_point_from_string(self, entry_point_string: str) -> EntryPoint: else: raise ValueError(f'invalid entry point string format: {entry_point_string}') - # If there is a factory for the entry point group, use that, otherwise use ``get_entry_point`` try: - get_entry_point_partial = functools.partial(self._factory_mapping[group], load=False) - except KeyError: - get_entry_point_partial = functools.partial(get_entry_point, group) - - try: - return get_entry_point_partial(name) + return get_entry_point(group, name) except exceptions.EntryPointError as exception: raise ValueError(exception) @@ -241,7 +233,8 @@ def validate_entry_point_group(self, group: str) -> None: if group not in self.groups: raise ValueError(f'entry point group `{group}` is not supported by this parameter.') - def convert(self, value: t.Any, param: click.Parameter, ctx: click.Context) -> t.Any: + def convert(self, value: t.Any, param: click.Parameter | None, + ctx: click.Context | None) -> t.Union[EntryPoint, t.Any]: """ Convert the string value to an entry point instance, if the value can be successfully parsed into an actual entry point. Will raise click.BadParameter if validation fails. diff --git a/aiida/manage/configuration/settings.py b/aiida/manage/configuration/settings.py index 9c75d284eb..6c4f9e1889 100644 --- a/aiida/manage/configuration/settings.py +++ b/aiida/manage/configuration/settings.py @@ -10,7 +10,7 @@ """Base settings required for the configuration of an AiiDA instance.""" import os import pathlib -import typing as t +from typing import Optional import warnings DEFAULT_UMASK = 0o0077 @@ -24,10 +24,11 @@ DEFAULT_DAEMON_LOG_DIR_NAME = 'log' DEFAULT_ACCESS_CONTROL_DIR_NAME = 'access' -AIIDA_CONFIG_FOLDER: t.Optional[pathlib.Path] = None -DAEMON_DIR: t.Optional[pathlib.Path] = None -DAEMON_LOG_DIR: t.Optional[pathlib.Path] = None -ACCESS_CONTROL_DIR: t.Optional[pathlib.Path] = None +# Assign defaults which may be overriden in set_configuration_directory() below +AIIDA_CONFIG_FOLDER: pathlib.Path = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME +DAEMON_DIR: pathlib.Path = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME +DAEMON_LOG_DIR: pathlib.Path = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME +ACCESS_CONTROL_DIR: pathlib.Path = AIIDA_CONFIG_FOLDER / DEFAULT_ACCESS_CONTROL_DIR_NAME def create_instance_directories() -> None: @@ -66,7 +67,7 @@ def create_instance_directories() -> None: os.umask(umask) -def set_configuration_directory(aiida_config_folder: t.Optional[pathlib.Path] = None) -> None: +def set_configuration_directory(aiida_config_folder: Optional[pathlib.Path] = None) -> None: """Determine location of configuration directory, set related global variables and create instance directories. The location of the configuration folder will be determined and optionally created following these heuristics: @@ -105,10 +106,7 @@ def set_configuration_directory(aiida_config_folder: t.Optional[pathlib.Path] = # If the directory exists, we leave it set and break the loop if AIIDA_CONFIG_FOLDER.is_dir(): break - - else: - # The `AIIDA_PATH` variable is not set so use the default path and try to create it if it does not exist - AIIDA_CONFIG_FOLDER = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME + # else: we use the default path defined at the top of this module DAEMON_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME DAEMON_LOG_DIR = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME From 5d398c6e31e9f7e0a3584c173764d8cc8cae639f Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 19:44:16 +0100 Subject: [PATCH 26/31] Move _get_valid_groups --- aiida/cmdline/params/types/plugin.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/aiida/cmdline/params/types/plugin.py b/aiida/cmdline/params/types/plugin.py index ca919e1b90..eecc6b0755 100644 --- a/aiida/cmdline/params/types/plugin.py +++ b/aiida/cmdline/params/types/plugin.py @@ -78,8 +78,9 @@ def __init__(self, group: str | tuple[str] | None = None, load: bool = False, *a super().__init__(*args, **kwargs) - def _get_valid_groups(self) -> tuple[str, ...]: - """Get allowed groups for this instance""" + @functools.cached_property + def groups(self) -> tuple[str, ...]: + """Returns a tuple of valid groups for this instance""" group = self._input_group valid_entry_point_groups = get_entry_point_groups() @@ -108,10 +109,6 @@ def _get_valid_groups(self) -> tuple[str, ...]: return tuple(groups) - @functools.cached_property - def groups(self) -> tuple[str, ...]: - return self._get_valid_groups() - @functools.cached_property def _entry_points(self) -> list[tuple[str, EntryPoint]]: return [(group, entry_point) for group in self.groups for entry_point in get_entry_points(group)] From 347f1b074865f710d04fe0b8fc1e605144aa2f42 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 20:18:32 +0100 Subject: [PATCH 27/31] ugh, stupid tui docs --- docs/source/reference/command_line.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/source/reference/command_line.rst b/docs/source/reference/command_line.rst index fcc0f4e17f..c3ae6e2f7c 100644 --- a/docs/source/reference/command_line.rst +++ b/docs/source/reference/command_line.rst @@ -571,6 +571,21 @@ Below is a list with all available subcommands. version Print the current version of the storage schema. +.. _reference:command-line:verdi-tui: + +``verdi tui`` +------------- + +.. code:: console + + Usage: [OPTIONS] + + Open Textual TUI. + + Options: + --help Show this message and exit. + + .. _reference:command-line:verdi-user: ``verdi user`` From a3ca765fb1c7f94335fd748719f6735a6090a628 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 20 Oct 2023 22:58:55 +0100 Subject: [PATCH 28/31] fix test_environment_variable_not_set --- aiida/manage/configuration/settings.py | 4 +++- tests/manage/configuration/test_config.py | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aiida/manage/configuration/settings.py b/aiida/manage/configuration/settings.py index 6c4f9e1889..0615093617 100644 --- a/aiida/manage/configuration/settings.py +++ b/aiida/manage/configuration/settings.py @@ -106,7 +106,9 @@ def set_configuration_directory(aiida_config_folder: Optional[pathlib.Path] = No # If the directory exists, we leave it set and break the loop if AIIDA_CONFIG_FOLDER.is_dir(): break - # else: we use the default path defined at the top of this module + else: + # The `AIIDA_PATH` variable is not set so use the default path and try to create it if it does not exist + AIIDA_CONFIG_FOLDER = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME DAEMON_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME DAEMON_LOG_DIR = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME diff --git a/tests/manage/configuration/test_config.py b/tests/manage/configuration/test_config.py index 399c111eea..613a346617 100644 --- a/tests/manage/configuration/test_config.py +++ b/tests/manage/configuration/test_config.py @@ -23,18 +23,16 @@ @pytest.fixture def cache_aiida_path_variable(): """Fixture that will store the ``AIIDA_PATH`` environment variable and restore it after the yield.""" - aiida_path_original = os.environ.get(settings.DEFAULT_AIIDA_PATH_VARIABLE, None) - - try: - yield - finally: - if aiida_path_original is not None: - os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = aiida_path_original - else: - try: - del os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] - except KeyError: - pass + aiida_path_original = os.environ.get(settings.DEFAULT_AIIDA_PATH_VARIABLE) + + yield + if aiida_path_original is not None: + os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] = aiida_path_original + else: + try: + del os.environ[settings.DEFAULT_AIIDA_PATH_VARIABLE] + except KeyError: + pass # Make sure to reset the global variables set by the following call that are dependent on the environment variable # ``DEFAULT_AIIDA_PATH_VARIABLE``. It may have been changed by a test using this fixture. From 76b480e0377cc465a4bc9d7809fe0270d4919125 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 21 Oct 2023 20:11:57 +0100 Subject: [PATCH 29/31] review + enable typing for cmdline/groups/dynamic.py --- .pre-commit-config.yaml | 1 - aiida/cmdline/groups/dynamic.py | 23 ++++++++++++++--------- aiida/cmdline/groups/verdi.py | 4 ++-- aiida/cmdline/params/types/identifier.py | 2 +- aiida/manage/configuration/settings.py | 5 +++-- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ea8a03175..130fe378dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,7 +93,6 @@ repos: aiida/cmdline/commands/cmd_node.py| aiida/cmdline/commands/cmd_shell.py| aiida/cmdline/commands/cmd_storage.py| - aiida/cmdline/groups/dynamic.py| aiida/cmdline/params/options/commands/setup.py| aiida/cmdline/params/options/interactive.py| aiida/cmdline/params/options/main.py| diff --git a/aiida/cmdline/groups/dynamic.py b/aiida/cmdline/groups/dynamic.py index 6503b031ba..f01f640813 100644 --- a/aiida/cmdline/groups/dynamic.py +++ b/aiida/cmdline/groups/dynamic.py @@ -44,14 +44,16 @@ def cmd_create(): def __init__( self, - command, + command: t.Callable, entry_point_group: str, entry_point_name_filter: str = r'.*', shared_options: list[click.Option] | None = None, **kwargs ): super().__init__(**kwargs) - self.command = command + # NOTE: We should probably fix this properly, mypy complains: + # "error: Cannot assign to a method" + self.command = command # type: ignore[assignment] self.entry_point_group = entry_point_group self.entry_point_name_filter = entry_point_name_filter self.factory = ENTRY_POINT_GROUP_FACTORY_MAPPING[entry_point_group] @@ -69,7 +71,7 @@ def list_commands(self, ctx: click.Context) -> list[str]: ]) return sorted(commands) - def get_command(self, ctx: click.Context, cmd_name) -> t.Any: + def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None: """Return the command with the given name. :param ctx: The :class:`click.Context`. @@ -77,19 +79,19 @@ def get_command(self, ctx: click.Context, cmd_name) -> t.Any: :returns: The :class:`click.Command`. """ try: - command = self.create_command(ctx, cmd_name) + command: click.Command | None = self.create_command(ctx, cmd_name) except exceptions.EntryPointError: command = super().get_command(ctx, cmd_name) return command - def create_command(self, ctx: click.Context, entry_point: str) -> t.Any: + def create_command(self, ctx: click.Context, entry_point: str) -> click.Command: """Create a subcommand for the given ``entry_point``.""" cls = self.factory(entry_point) command = functools.partial(self.command, ctx, cls) command.__doc__ = cls.__doc__ return click.command(entry_point)(self.create_options(entry_point)(command)) - def create_options(self, entry_point): + def create_options(self, entry_point: str) -> t.Callable: """Create the option decorators for the command function for the given entry point. :param entry_point: The entry point. @@ -116,15 +118,18 @@ def apply_options(func): return apply_options - def list_options(self, entry_point): + def list_options(self, entry_point: str) -> list: """Return the list of options that should be applied to the command for the given entry point. :param entry_point: The entry point. """ - return [self.create_option(*item) for item in self.factory(entry_point).get_cli_options().items()] + return [ + self.create_option(*item) + for item in self.factory(entry_point).get_cli_options().items() # type: ignore[union-attr] + ] @staticmethod - def create_option(name, spec): + def create_option(name, spec: dict) -> t.Callable[[t.Any], t.Any]: """Create a click option from a name and a specification.""" spec = copy.deepcopy(spec) diff --git a/aiida/cmdline/groups/verdi.py b/aiida/cmdline/groups/verdi.py index 91fcb88ba1..951978f88f 100644 --- a/aiida/cmdline/groups/verdi.py +++ b/aiida/cmdline/groups/verdi.py @@ -83,7 +83,7 @@ class VerdiCommandGroup(click.Group): context_class = VerdiContext @staticmethod - def add_verbosity_option(cmd: click.Command): + def add_verbosity_option(cmd: click.Command) -> click.Command: """Apply the ``verbosity`` option to the command, which is common to all ``verdi`` commands.""" # Only apply the option if it hasn't been already added in a previous call. if 'verbosity' not in [param.name for param in cmd.params]: @@ -91,7 +91,7 @@ def add_verbosity_option(cmd: click.Command): return cmd - def fail_with_suggestions(self, ctx: click.Context, cmd_name: str): + def fail_with_suggestions(self, ctx: click.Context, cmd_name: str) -> None: """Fail the command while trying to suggest commands to resemble the requested ``cmd_name``.""" # We might get better results with the Levenshtein distance or more advanced methods implemented in FuzzyWuzzy # or similar libs, but this is an easy win for now. diff --git a/aiida/cmdline/params/types/identifier.py b/aiida/cmdline/params/types/identifier.py index 4d521661f7..17358d5763 100644 --- a/aiida/cmdline/params/types/identifier.py +++ b/aiida/cmdline/params/types/identifier.py @@ -48,7 +48,7 @@ def __init__(self, sub_classes: tuple[str, ...] | None = None): To prevent having to load the database environment at import time, the actual loading of the entry points is deferred until the call to `convert` is made. This is to keep the command line autocompletion light - and responsive. We also postpone the validation of entry point strings for the same reason. + and responsive. The validation of entry point strings is also postponed for the same reason. :param sub_classes: a tuple of entry point strings that can narrow the set of orm classes that values will be mapped upon. These classes have to be strict sub classes of the base orm class defined diff --git a/aiida/manage/configuration/settings.py b/aiida/manage/configuration/settings.py index 0615093617..0f68c4339c 100644 --- a/aiida/manage/configuration/settings.py +++ b/aiida/manage/configuration/settings.py @@ -8,9 +8,10 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Base settings required for the configuration of an AiiDA instance.""" +from __future__ import annotations + import os import pathlib -from typing import Optional import warnings DEFAULT_UMASK = 0o0077 @@ -67,7 +68,7 @@ def create_instance_directories() -> None: os.umask(umask) -def set_configuration_directory(aiida_config_folder: Optional[pathlib.Path] = None) -> None: +def set_configuration_directory(aiida_config_folder: pathlib.Path | None = None) -> None: """Determine location of configuration directory, set related global variables and create instance directories. The location of the configuration folder will be determined and optionally created following these heuristics: From bec366280eef7590e12972bbb5ed27d195b4f6cd Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 21 Oct 2023 23:36:18 +0100 Subject: [PATCH 30/31] Improve typing --- aiida/cmdline/groups/dynamic.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aiida/cmdline/groups/dynamic.py b/aiida/cmdline/groups/dynamic.py index f01f640813..89643ef514 100644 --- a/aiida/cmdline/groups/dynamic.py +++ b/aiida/cmdline/groups/dynamic.py @@ -51,9 +51,7 @@ def __init__( **kwargs ): super().__init__(**kwargs) - # NOTE: We should probably fix this properly, mypy complains: - # "error: Cannot assign to a method" - self.command = command # type: ignore[assignment] + self._command = command self.entry_point_group = entry_point_group self.entry_point_name_filter = entry_point_name_filter self.factory = ENTRY_POINT_GROUP_FACTORY_MAPPING[entry_point_group] @@ -87,7 +85,7 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None def create_command(self, ctx: click.Context, entry_point: str) -> click.Command: """Create a subcommand for the given ``entry_point``.""" cls = self.factory(entry_point) - command = functools.partial(self.command, ctx, cls) + command = functools.partial(self._command, ctx, cls) command.__doc__ = cls.__doc__ return click.command(entry_point)(self.create_options(entry_point)(command)) From 6930e7988b80cfe100f5a16edb94c1e224dab694 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Sun, 22 Oct 2023 08:36:43 +0200 Subject: [PATCH 31/31] Fix docs --- docs/source/nitpick-exceptions | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/nitpick-exceptions b/docs/source/nitpick-exceptions index 8027da86dd..0c3a0eea15 100644 --- a/docs/source/nitpick-exceptions +++ b/docs/source/nitpick-exceptions @@ -11,6 +11,8 @@ py:class NoneType py:class MappingType py:class AbstractContextManager py:class BinaryIO +py:class EntryPoint +py:class EntryPoints py:class IO py:class Path py:class str | list[str] @@ -191,6 +193,8 @@ py:class sphinx.util.docutils.SphinxDirective py:class ModuleAnalyzer py:class BuildEnvironment +py:meth tabulate.tabulate + py:class yaml.Dumper py:class yaml.Loader py:class yaml.dumper.Dumper