Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lazy entry points in verdi #6153

Merged
merged 34 commits into from
Oct 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
ebf9fc0
WIP: Lazy entry points in verdi
danielhollas Oct 14, 2023
3b5dde9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 17, 2023
8559c37
Try running tests
danielhollas Oct 17, 2023
47aa3ba
Fix EntryPoint imports
danielhollas Oct 17, 2023
aebdb2a
Revert breaking changes to see if tests pass
danielhollas Oct 17, 2023
82cefcd
I guess this does not work?
danielhollas Oct 18, 2023
a0dd26f
One more
danielhollas Oct 18, 2023
15b498f
Break
danielhollas Oct 18, 2023
2d6b8b0
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 18, 2023
9e01860
Remove click from aiida.orm
danielhollas Oct 18, 2023
a66ba9c
initialize to empty lists
danielhollas Oct 18, 2023
2f8697a
Lazy import tabulate, ~6ms
danielhollas Oct 18, 2023
ee62868
Lazy import inspect, ~7ms
danielhollas Oct 18, 2023
5c4488d
Lazy imports of stdlib, ~15ms
danielhollas Oct 18, 2023
fd15352
One more temporary fix for tests
danielhollas Oct 18, 2023
57e1f26
Lazy PluginParamType
danielhollas Oct 18, 2023
4174c80
lazy IdentifierParamType
danielhollas Oct 18, 2023
615a0aa
Update tests
danielhollas Oct 18, 2023
f036805
Merge branch 'main' into verdi/lazy-entry-points
danielhollas Oct 18, 2023
82b950a
Update aiida/cmdline/utils/echo.py
danielhollas Oct 20, 2023
ceb3ec9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 20, 2023
8d88334
Minor refactoring and typing fix in configuration.py
danielhollas Oct 20, 2023
7a2cc44
review cleanup
danielhollas Oct 20, 2023
2b82927
Update docstring
danielhollas Oct 20, 2023
d4b2491
Optimization and typing in params/types/plugin.py
danielhollas Oct 20, 2023
3e945c0
Enable typing for params/types/plugin.py|identifier.py and aiida/mana…
danielhollas Oct 20, 2023
5d398c6
Move _get_valid_groups
danielhollas Oct 20, 2023
347f1b0
ugh, stupid tui docs
danielhollas Oct 20, 2023
a3ca765
fix test_environment_variable_not_set
danielhollas Oct 20, 2023
4b0d8aa
Merge branch 'main' into verdi/lazy-entry-points
danielhollas Oct 20, 2023
76b480e
review + enable typing for cmdline/groups/dynamic.py
danielhollas Oct 21, 2023
bec3662
Improve typing
danielhollas Oct 21, 2023
d356545
Merge branch 'main' into verdi/lazy-entry-points
danielhollas Oct 21, 2023
6930e79
Fix docs
sphuber Oct 22, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,11 @@ repos:
aiida/cmdline/commands/cmd_node.py|
aiida/cmdline/commands/cmd_shell.py|
aiida/cmdline/commands/cmd_storage.py|
aiida/cmdline/groups/dynamic.py|
aiida/cmdline/params/options/commands/setup.py|
aiida/cmdline/params/options/interactive.py|
aiida/cmdline/params/options/main.py|
aiida/cmdline/params/options/multivalue.py|
aiida/cmdline/params/types/group.py|
aiida/cmdline/params/types/plugin.py|
aiida/cmdline/utils/ascii_vis.py|
aiida/cmdline/utils/common.py|
aiida/cmdline/utils/echo.py|
Expand All @@ -115,7 +113,6 @@ repos:
aiida/manage/configuration/__init__.py|
aiida/manage/configuration/config.py|
aiida/manage/configuration/profile.py|
aiida/manage/configuration/settings.py|
aiida/manage/external/rmq/launcher.py|
aiida/manage/tests/main.py|
aiida/manage/tests/pytest_fixtures.py|
Expand Down
1 change: 1 addition & 0 deletions aiida/cmdline/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
'echo_info',
'echo_report',
'echo_success',
'echo_tabulate',
'echo_warning',
'format_call_graph',
'is_verbose',
Expand Down
7 changes: 3 additions & 4 deletions aiida/cmdline/commands/cmd_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,12 @@
from functools import partial

import click
import tabulate

from aiida.cmdline.commands.cmd_verdi import verdi
from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup
from aiida.cmdline.params import arguments, options, types
from aiida.cmdline.params.options.commands import code as options_code
from aiida.cmdline.utils import echo
from aiida.cmdline.utils import echo, echo_tabulate
from aiida.cmdline.utils.decorators import deprecated_command, with_dbenv
from aiida.common import exceptions

Expand Down Expand Up @@ -232,7 +231,7 @@ def show(code):
if is_verbose():
table.append(['Calculations', len(code.base.links.get_outgoing().all())])

echo.echo(tabulate.tabulate(table))
echo_tabulate(table)


@verdi_code.command()
Expand Down Expand Up @@ -419,7 +418,7 @@ def code_list(computer, default_calc_job_plugin, all_entries, all_users, raw, sh
row.append('@'.join(str(result[entity][projection]) for entity, projection in VALID_PROJECTIONS[key]))
table.append(row)

echo.echo(tabulate.tabulate(table, headers=headers, tablefmt=table_format))
echo_tabulate(table, headers=headers, tablefmt=table_format)

if not raw:
echo.echo_report('\nUse `verdi code show IDENTIFIER` to see details for a code', prefix=False)
7 changes: 3 additions & 4 deletions aiida/cmdline/commands/cmd_computer.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,11 @@
from math import isclose

import click
import tabulate

from aiida.cmdline.commands.cmd_verdi import VerdiCommandGroup, verdi
from aiida.cmdline.params import arguments, options
from aiida.cmdline.params.options.commands import computer as options_computer
from aiida.cmdline.utils import echo
from aiida.cmdline.utils import echo, echo_tabulate
from aiida.cmdline.utils.decorators import with_dbenv
from aiida.common.exceptions import EntryPointError, ValidationError
from aiida.plugins.entry_point import get_entry_point_names
Expand Down Expand Up @@ -461,7 +460,7 @@ def computer_show(computer):
['Prepend text', computer.get_prepend_text()],
['Append text', computer.get_append_text()],
]
echo.echo(tabulate.tabulate(table))
echo_tabulate(table)


@verdi_computer.command('relabel')
Expand Down Expand Up @@ -697,4 +696,4 @@ def computer_config_show(computer, user, defaults, as_option_string):
table.append((f'* {name}', config[name]))
else:
table.append((f'* {name}', '-'))
echo.echo(tabulate.tabulate(table, tablefmt='plain'))
echo_tabulate(table, tablefmt='plain')
17 changes: 9 additions & 8 deletions aiida/cmdline/commands/cmd_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,13 @@
###########################################################################
"""`verdi node` command."""
import pathlib
import shutil

import click
import tabulate

from aiida.cmdline.commands.cmd_verdi import verdi
from aiida.cmdline.params import arguments, options
from aiida.cmdline.params.types.plugin import PluginParamType
from aiida.cmdline.utils import decorators, echo, multi_line_input
from aiida.cmdline.utils import decorators, echo, echo_tabulate, multi_line_input
from aiida.cmdline.utils.decorators import with_dbenv
from aiida.common import exceptions, timezone
from aiida.common.links import GraphTraversalRules
Expand All @@ -43,6 +41,7 @@ def repo_cat(node, relative_path):
For ``SinglefileData`` nodes, the `RELATIVE_PATH` does not have to be specified as it is determined automatically.
"""
import errno
import shutil
import sys

from aiida.orm import SinglefileData
Expand Down Expand Up @@ -92,6 +91,8 @@ def repo_dump(node, output_directory):
The output directory should not exist. If it does, the command
will abort.
"""
import shutil

from aiida.repository import FileType

output_directory = pathlib.Path(output_directory)
Expand Down Expand Up @@ -146,9 +147,9 @@ def node_label(nodes, label, raw, force):
table.append([node.pk, node.label])

if raw:
echo.echo(tabulate.tabulate(table, tablefmt='plain'))
echo_tabulate(table, tablefmt='plain')
else:
echo.echo(tabulate.tabulate(table, headers=['ID', 'Label']))
echo_tabulate(table, headers=['ID', 'Label'])

else:
if not force:
Expand Down Expand Up @@ -180,9 +181,9 @@ def node_description(nodes, description, force, raw):
table.append([node.pk, node.description])

if raw:
echo.echo(tabulate.tabulate(table, tablefmt='plain'))
echo_tabulate(table, tablefmt='plain')
else:
echo.echo(tabulate.tabulate(table, headers=['ID', 'Description']))
echo_tabulate(table, headers=['ID', 'Description'])

else:
if not force:
Expand Down Expand Up @@ -229,7 +230,7 @@ def node_show(nodes, print_groups):
table = [(gr['groups']['id'], gr['groups']['label'], gr['groups']['type_string']) for gr in res]
table.sort()

echo.echo(tabulate.tabulate(table, headers=['PK', 'Label', 'Group type']))
echo_tabulate(table, headers=['PK', 'Label', 'Group type'])


def echo_node_dict(nodes, keys, fmt, identifier, raw, use_attrs=True):
Expand Down
3 changes: 2 additions & 1 deletion aiida/cmdline/commands/cmd_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
# For further information please visit http://www.aiida.net #
###########################################################################
"""Command for `verdi plugins`."""
import inspect

import click

Expand All @@ -27,6 +26,8 @@ def verdi_plugin():
@click.argument('entry_point', type=click.STRING, required=False)
def plugin_list(entry_point_group, entry_point):
"""Display a list of all available plugins."""
import inspect

from aiida.cmdline.utils.common import print_process_info
from aiida.common import EntryPointError
from aiida.engine import Process
Expand Down
5 changes: 2 additions & 3 deletions aiida/cmdline/commands/cmd_rabbitmq.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,11 @@
import typing as t

import click
import tabulate
import wrapt

from aiida.cmdline.commands.cmd_devel import verdi_devel
from aiida.cmdline.params import arguments, options
from aiida.cmdline.utils import decorators, echo
from aiida.cmdline.utils import decorators, echo, echo_tabulate

if t.TYPE_CHECKING:
import kiwipy.rmq
Expand Down Expand Up @@ -192,7 +191,7 @@ def cmd_queues_list(client, project, raw, filter_name):

headers = [name.capitalize() for name in project] if not raw else []
tablefmt = None if not raw else 'plain'
echo.echo(tabulate.tabulate(output, headers=headers, tablefmt=tablefmt))
echo_tabulate(output, headers=headers, tablefmt=tablefmt)


@cmd_queues.command('create')
Expand Down
26 changes: 15 additions & 11 deletions aiida/cmdline/groups/dynamic.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import copy
import functools
import re
import typing as t

import click

Expand Down Expand Up @@ -43,20 +44,20 @@ def cmd_create():

def __init__(
self,
command,
command: t.Callable,
entry_point_group: str,
entry_point_name_filter: str = r'.*',
shared_options: list[click.Option] | None = None,
**kwargs
):
super().__init__(**kwargs)
self.command = command
self._command = command
self.entry_point_group = entry_point_group
self.entry_point_name_filter = entry_point_name_filter
self.factory = ENTRY_POINT_GROUP_FACTORY_MAPPING[entry_point_group]
self.shared_options = shared_options

def list_commands(self, ctx) -> list[str]:
def list_commands(self, ctx: click.Context) -> list[str]:
"""Return the sorted list of subcommands for this group.

:param ctx: The :class:`click.Context`.
Expand All @@ -68,27 +69,27 @@ def list_commands(self, ctx) -> list[str]:
])
return sorted(commands)

def get_command(self, ctx, cmd_name):
def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
"""Return the command with the given name.

:param ctx: The :class:`click.Context`.
:param cmd_name: The name of the command.
:returns: The :class:`click.Command`.
"""
try:
command = self.create_command(ctx, cmd_name)
command: click.Command | None = self.create_command(ctx, cmd_name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

create_command returns click.Command so why is command typed as click.Command | None

Suggested change
command: click.Command | None = self.create_command(ctx, cmd_name)
command: click.Command = self.create_command(ctx, cmd_name)

Copy link
Collaborator Author

@danielhollas danielhollas Oct 21, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but the super().get_command(ctx, cmd_name) below returns click.Command | None to the same variable. If I remove this annotation mypy complains

error: Incompatible types in assignment (expression has type "Optional[Command]", variable has type "Command")  [assignment]

I don't know why mypy complains, it should be able to infer this from the return type. Is there a better solution to this?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm weird, let's leave it then for now.

except exceptions.EntryPointError:
command = super().get_command(ctx, cmd_name)
return command

def create_command(self, ctx, entry_point):
def create_command(self, ctx: click.Context, entry_point: str) -> click.Command:
"""Create a subcommand for the given ``entry_point``."""
cls = self.factory(entry_point)
command = functools.partial(self.command, ctx, cls)
command = functools.partial(self._command, ctx, cls)
command.__doc__ = cls.__doc__
return click.command(entry_point)(self.create_options(entry_point)(command))

def create_options(self, entry_point):
def create_options(self, entry_point: str) -> t.Callable:
"""Create the option decorators for the command function for the given entry point.

:param entry_point: The entry point.
Expand All @@ -115,15 +116,18 @@ def apply_options(func):

return apply_options

def list_options(self, entry_point):
def list_options(self, entry_point: str) -> list:
"""Return the list of options that should be applied to the command for the given entry point.

:param entry_point: The entry point.
"""
return [self.create_option(*item) for item in self.factory(entry_point).get_cli_options().items()]
return [
self.create_option(*item)
for item in self.factory(entry_point).get_cli_options().items() # type: ignore[union-attr]
]

@staticmethod
def create_option(name, spec):
def create_option(name, spec: dict) -> t.Callable[[t.Any], t.Any]:
"""Create a click option from a name and a specification."""
spec = copy.deepcopy(spec)

Expand Down
4 changes: 2 additions & 2 deletions aiida/cmdline/groups/verdi.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,15 @@ class VerdiCommandGroup(click.Group):
context_class = VerdiContext

@staticmethod
def add_verbosity_option(cmd: click.Command):
def add_verbosity_option(cmd: click.Command) -> click.Command:
"""Apply the ``verbosity`` option to the command, which is common to all ``verdi`` commands."""
# Only apply the option if it hasn't been already added in a previous call.
if 'verbosity' not in [param.name for param in cmd.params]:
cmd = options.VERBOSITY()(cmd)

return cmd

def fail_with_suggestions(self, ctx: click.Context, cmd_name: str):
def fail_with_suggestions(self, ctx: click.Context, cmd_name: str) -> None:
"""Fail the command while trying to suggest commands to resemble the requested ``cmd_name``."""
# We might get better results with the Levenshtein distance or more advanced methods implemented in FuzzyWuzzy
# or similar libs, but this is an easy win for now.
Expand Down
5 changes: 4 additions & 1 deletion aiida/cmdline/params/options/commands/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
"""Reusable command line interface options for the setup commands."""
import functools
import getpass
import hashlib

import click

Expand Down Expand Up @@ -94,6 +93,8 @@ def get_quicksetup_database_name(ctx, param, value): # pylint: disable=unused-a
:param ctx: click context which should contain the contextual parameters
:return: the database name
"""
import hashlib

if value is not None:
return value

Expand All @@ -114,6 +115,8 @@ def get_quicksetup_username(ctx, param, value): # pylint: disable=unused-argume
:param ctx: click context which should contain the contextual parameters
:return: the username
"""
import hashlib

if value is not None:
return value

Expand Down
Loading
Loading