From a7919ecb4f0796659cab0f69a6068bf63d1b2a62 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Mon, 13 Jan 2025 21:17:26 -0500 Subject: [PATCH 1/6] Ensure pypi-normalized names when fetching from npe2api --- data-workflows/plugin/processor.py | 13 +++++++++++-- .../src/nhcommons/utils/pypi_adapter.py | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/data-workflows/plugin/processor.py b/data-workflows/plugin/processor.py index 27d44fcef..4f59e6d72 100644 --- a/data-workflows/plugin/processor.py +++ b/data-workflows/plugin/processor.py @@ -42,10 +42,11 @@ def _is_new_plugin(plugin_version_pair): pypi_plugin_version = pypi_latest_plugins.get(name) if pypi_plugin_version == version: continue + if pypi_plugin_version is None and is_plugin_active(name, version): logger.info( f"Skipping marking plugin={name} version={version} stale as the " - f"plugin is still active in npe2api" + "plugin is still active in npe2api" ) continue @@ -55,8 +56,16 @@ def _is_new_plugin(plugin_version_pair): version=version, plugin_metadata_type=PluginMetadataType.PYPI, ) + if pypi_plugin_version is None: - zulip.plugin_no_longer_on_hub(name) + logger.info(f"Plugin no longer on hub: plugin={name}") + if name == pypi_adapter.pypi_name_normalize(name): + zulip.plugin_no_longer_on_hub(name) + else: + logger.info( + f"Skipping zulip removal notification plugin={name}, " + "name is not normalized, so likely a duplicate plugin." + ) def _update_for_new_plugin(name: str, version: str, old_version: Optional[str]) -> None: diff --git a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py index 921b61a00..decc664f1 100644 --- a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py @@ -17,6 +17,15 @@ logger = logging.getLogger(__name__) +def pypi_name_normalize(name: str) -> str: + """ + Normalize the plugin name to lowercase and replace special characters with hyphen. + + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + def get_all_plugins() -> Dict[str, str]: """ Query npe2api to get all plugins. @@ -24,13 +33,18 @@ def get_all_plugins() -> Dict[str, str]: Now we use the npe2api to get the list of plugins, which uses the public BigQuery pypi metadata as a source of truth. + This also normalizes the plugins names to match PyPI. + The previous implementation was broken by anti-scraping changes to PyPI. :returns: all plugin names and latest version """ logger.info("Getting all napari plugins from npe2api") packages = get_request(_NPE2API_URL + "/plugins").json() logger.info(f"Total number of napari plugins fetched={len(packages)}") - return packages + return { + pypi_name_normalize(name): version + for name, version in packages.items() + } def get_plugin_pypi_metadata(plugin: str, version: str) -> Dict[str, Any]: From 4cdeed0d41845657b3d5e24c2b347f06b107c01f Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 14 Jan 2025 11:01:02 -0500 Subject: [PATCH 2/6] Fix tests, cleanup --- data-workflows/plugin/processor.py | 33 +++++++++++-------- .../tests/utils/test_pypi_adapter.py | 5 ++- .../src/nhcommons/utils/pypi_adapter.py | 4 +-- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/data-workflows/plugin/processor.py b/data-workflows/plugin/processor.py index 4f59e6d72..7721407e8 100644 --- a/data-workflows/plugin/processor.py +++ b/data-workflows/plugin/processor.py @@ -37,8 +37,24 @@ def _is_new_plugin(plugin_version_pair): futures.wait(update_futures, return_when="ALL_COMPLETED") + def _mark_version_stale(name, version) -> None: + logger.info(f"Updating old plugin={name} version={version}") + put_plugin_metadata( + plugin=name, + version=version, + plugin_metadata_type=PluginMetadataType.PYPI, + ) + # update for removed plugins and existing older version of plugins for name, version in dynamo_latest_plugins.items(): + if name != (normalized := pypi_adapter.normalize(name)): + logger.info( + f"Found non-normalized name, will be removed plugin={name} " + f"(normalized={normalized})" + ) + _mark_version_stale(name, version) + continue + pypi_plugin_version = pypi_latest_plugins.get(name) if pypi_plugin_version == version: continue @@ -50,22 +66,11 @@ def _is_new_plugin(plugin_version_pair): ) continue - logger.info(f"Updating old plugin={name} version={version}") - put_plugin_metadata( - plugin=name, - version=version, - plugin_metadata_type=PluginMetadataType.PYPI, - ) + _mark_version_stale(name, version) if pypi_plugin_version is None: - logger.info(f"Plugin no longer on hub: plugin={name}") - if name == pypi_adapter.pypi_name_normalize(name): - zulip.plugin_no_longer_on_hub(name) - else: - logger.info( - f"Skipping zulip removal notification plugin={name}, " - "name is not normalized, so likely a duplicate plugin." - ) + logger.info(f"Plugin={name} will be removed from hub") + zulip.plugin_no_longer_on_hub(name) def _update_for_new_plugin(name: str, version: str, old_version: Optional[str]) -> None: diff --git a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py index ca2f5a0e3..8a873bc69 100644 --- a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py @@ -149,7 +149,10 @@ def _mocked_requests_get(self, *args, **kwargs): return MockResponse(status_code=requests.codes.not_found) def test_get_all_plugins(self): - expected = {plugin[0]: plugin[1] for plugin in plugins()} + expected = { + pypi_adapter.normalize(plugin[0]): plugin[1] + for plugin in plugins() + } assert expected == pypi_adapter.get_all_plugins() @pytest.mark.parametrize( diff --git a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py index decc664f1..d5d39187d 100644 --- a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def pypi_name_normalize(name: str) -> str: +def normalize(name: str) -> str: """ Normalize the plugin name to lowercase and replace special characters with hyphen. @@ -42,7 +42,7 @@ def get_all_plugins() -> Dict[str, str]: packages = get_request(_NPE2API_URL + "/plugins").json() logger.info(f"Total number of napari plugins fetched={len(packages)}") return { - pypi_name_normalize(name): version + normalize(name): version for name, version in packages.items() } From 4d8efa57cb2acdfe8ede1098b0c98cf27d74170b Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 14 Jan 2025 11:04:34 -0500 Subject: [PATCH 3/6] Blacken --- .../src/nhcommons/tests/utils/test_pypi_adapter.py | 7 ++++--- napari-hub-commons/src/nhcommons/utils/pypi_adapter.py | 5 +---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py index 8a873bc69..e3ab6712e 100644 --- a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py @@ -139,7 +139,9 @@ def setup_method(self, monkeypatch): def _mocked_requests_get(self, *args, **kwargs): if args[0] == "https://api.napari.org/api/plugins": - return MockResponse(content=json.dumps({name: version for name, version in plugins()})) + return MockResponse( + content=json.dumps({name: version for name, version in plugins()}) + ) elif args[0] == "https://pypi.org/pypi/napari-demo/json": return MockResponse(content=valid_pypi_data()) elif args[0] == "https://pypi.org/pypi/default-demo/json": @@ -150,8 +152,7 @@ def _mocked_requests_get(self, *args, **kwargs): def test_get_all_plugins(self): expected = { - pypi_adapter.normalize(plugin[0]): plugin[1] - for plugin in plugins() + pypi_adapter.normalize(plugin[0]): plugin[1] for plugin in plugins() } assert expected == pypi_adapter.get_all_plugins() diff --git a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py index d5d39187d..84b958f87 100644 --- a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py @@ -41,10 +41,7 @@ def get_all_plugins() -> Dict[str, str]: logger.info("Getting all napari plugins from npe2api") packages = get_request(_NPE2API_URL + "/plugins").json() logger.info(f"Total number of napari plugins fetched={len(packages)}") - return { - normalize(name): version - for name, version in packages.items() - } + return {normalize(name): version for name, version in packages.items()} def get_plugin_pypi_metadata(plugin: str, version: str) -> Dict[str, Any]: From bcbdd7310842f18c66100756a0599f86d7e26c29 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 14 Jan 2025 15:06:53 -0500 Subject: [PATCH 4/6] Remove 'is_active' check now that npe2api is source of truth --- data-workflows/plugin/classifier_adapter.py | 28 --------- data-workflows/plugin/processor.py | 23 ++----- .../plugin/tests/test_classifier_adapter.py | 61 ------------------- data-workflows/plugin/tests/test_processor.py | 53 ++++++++-------- 4 files changed, 30 insertions(+), 135 deletions(-) delete mode 100644 data-workflows/plugin/classifier_adapter.py delete mode 100644 data-workflows/plugin/tests/test_classifier_adapter.py diff --git a/data-workflows/plugin/classifier_adapter.py b/data-workflows/plugin/classifier_adapter.py deleted file mode 100644 index 70baa8121..000000000 --- a/data-workflows/plugin/classifier_adapter.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -from functools import cache - -from nhcommons.utils.adapter_helpers import GithubClientHelper - - -REPOSITORY = "https://github.com/napari/npe2api" -FILE_NAME = "public/classifiers.json" -LOGGER = logging.getLogger(__name__) - - -@cache -def _get_recent_query_data() -> dict: - github_client_helper = GithubClientHelper(REPOSITORY) - data = github_client_helper.get_file(FILE_NAME, "json") - if not data: - raise RuntimeError(f"Unable to fetch {FILE_NAME} from {REPOSITORY}") - return data - - -def is_plugin_active(name: str, version: str) -> bool: - try: - recent_query_update = _get_recent_query_data() - active_versions = set(recent_query_update.get("active", {}).get(name, [])) - return version in active_versions - except RuntimeError: - LOGGER.warning(f"Returning {name} {version} is active due to RuntimeError") - return True diff --git a/data-workflows/plugin/processor.py b/data-workflows/plugin/processor.py index 7721407e8..957575706 100644 --- a/data-workflows/plugin/processor.py +++ b/data-workflows/plugin/processor.py @@ -2,7 +2,6 @@ from concurrent import futures from typing import Optional -from plugin.classifier_adapter import is_plugin_active from plugin.lambda_adapter import LambdaAdapter from nhcommons.models.plugin_utils import PluginMetadataType from nhcommons.utils import pypi_adapter @@ -49,28 +48,18 @@ def _mark_version_stale(name, version) -> None: for name, version in dynamo_latest_plugins.items(): if name != (normalized := pypi_adapter.normalize(name)): logger.info( - f"Found non-normalized name, will be removed plugin={name} " + f"Found non-normalized name, plugin={name} will be removed" f"(normalized={normalized})" ) _mark_version_stale(name, version) continue pypi_plugin_version = pypi_latest_plugins.get(name) - if pypi_plugin_version == version: - continue - - if pypi_plugin_version is None and is_plugin_active(name, version): - logger.info( - f"Skipping marking plugin={name} version={version} stale as the " - "plugin is still active in npe2api" - ) - continue - - _mark_version_stale(name, version) - - if pypi_plugin_version is None: - logger.info(f"Plugin={name} will be removed from hub") - zulip.plugin_no_longer_on_hub(name) + if pypi_plugin_version != version: + _mark_version_stale(name, version) + if pypi_plugin_version is None: + logger.info(f"Plugin={name} will be removed from hub") + zulip.plugin_no_longer_on_hub(name) def _update_for_new_plugin(name: str, version: str, old_version: Optional[str]) -> None: diff --git a/data-workflows/plugin/tests/test_classifier_adapter.py b/data-workflows/plugin/tests/test_classifier_adapter.py deleted file mode 100644 index 4c9193008..000000000 --- a/data-workflows/plugin/tests/test_classifier_adapter.py +++ /dev/null @@ -1,61 +0,0 @@ -from unittest.mock import Mock, call - -import pytest - -import nhcommons.utils.adapter_helpers -import plugin.classifier_adapter - -REPOSITORY = "https://github.com/napari/npe2api" -FILE_NAME = "public/classifiers.json" - - -class TestClassifierAdapter: - @pytest.fixture(autouse=True) - def clear_cache(self): - plugin.classifier_adapter._get_recent_query_data.cache_clear() - - @pytest.fixture(autouse=True) - def github_client_helper(self, monkeypatch): - self._github_client_helper = Mock() - self._github_client_helper.get_file.side_effect = ( - lambda _, __: self._classifier_json - ) - self._github_client_helper_call = Mock( - spec=nhcommons.utils.adapter_helpers.GithubClientHelper, - return_value=self._github_client_helper, - ) - monkeypatch.setattr( - plugin.classifier_adapter, - "GithubClientHelper", - self._github_client_helper_call, - ) - - def test_handle_valid_query_data(self): - self._classifier_json = {"active": {"foo": ["1.0.0", "1.0.1", "1.0.2"]}} - assert True == plugin.classifier_adapter.is_plugin_active("foo", "1.0.2") - assert False == plugin.classifier_adapter.is_plugin_active("foo", "0.0.9") - assert False == plugin.classifier_adapter.is_plugin_active("bar", "1.0.4") - self._github_client_helper_call.assert_called_once_with(REPOSITORY) - self._github_client_helper.get_file.assert_called_once_with(FILE_NAME, "json") - - def test_handle_invalid_query_data(self): - self._classifier_json = {"inactive": []} - assert False == plugin.classifier_adapter.is_plugin_active("foo", "1.0.2") - assert False == plugin.classifier_adapter.is_plugin_active("bar", "1.0.2") - self._github_client_helper_call.assert_called_once_with(REPOSITORY) - self._github_client_helper.get_file.assert_called_once_with(FILE_NAME, "json") - - @pytest.mark.parametrize("init_classifier_json", [None, {}]) - def test_is_plugin_live_does_not_cache_error(self, init_classifier_json): - # When the classifier json is not a dict with values, _get_recent_query_data() - # should throw a RuntimeError - self._classifier_json = init_classifier_json - assert True == plugin.classifier_adapter.is_plugin_active("foo", "1.0.2") - self._classifier_json = {"active": {"foo": ["1.0.0", "1.0.1", "1.0.2"]}} - assert True == plugin.classifier_adapter.is_plugin_active("foo", "1.0.2") - self._github_client_helper_call.assert_has_calls( - [call(REPOSITORY), call(REPOSITORY)] - ) - self._github_client_helper.get_file.assert_has_calls( - [call(FILE_NAME, "json"), call(FILE_NAME, "json")] - ) diff --git a/data-workflows/plugin/tests/test_processor.py b/data-workflows/plugin/tests/test_processor.py index b2f49ba93..d5044c9e7 100644 --- a/data-workflows/plugin/tests/test_processor.py +++ b/data-workflows/plugin/tests/test_processor.py @@ -3,7 +3,6 @@ import pytest import plugin.lambda_adapter -import plugin.classifier_adapter import plugin.metadata from nhcommons.models.plugin_utils import PluginMetadataType as PMType @@ -69,15 +68,6 @@ def mock_lambda_adapter(self, monkeypatch) -> Mock: monkeypatch.setattr(processor, "LambdaAdapter", mock) return mock - @pytest.fixture - def mock_classifier_adapter(self, monkeypatch) -> Mock: - mock = Mock( - side_effect=lambda _, __: self._is_plugin_live, - spec=plugin.classifier_adapter, - ) - monkeypatch.setattr(processor, "is_plugin_active", mock) - return mock - @pytest.fixture def mock_zulip(self, monkeypatch) -> Mock: mock = Mock(spec=zulip) @@ -93,7 +83,6 @@ def setup( mock_get_existing_types, mock_get_formatted_metadata, mock_lambda_adapter, - mock_classifier_adapter, mock_zulip, ) -> None: self._get_latest_plugins = mock_get_latest_plugins @@ -102,7 +91,6 @@ def setup( self._get_existing_types = mock_get_existing_types self._get_formatted_metadata = mock_get_formatted_metadata self._lambda_adapter = mock_lambda_adapter - self._classifier_adapter = mock_classifier_adapter self._zulip = mock_zulip @pytest.fixture @@ -115,7 +103,6 @@ def _verify_calls( get_formatted_metadata_called: bool = False, lambda_invoked: bool = False, put_pm_calls: list = None, - classifier_adapter_not_called: bool = True, ) -> None: verify_call(True, self._get_latest_plugins, empty_call_list) verify_call(True, self._get_all_plugins, empty_call_list) @@ -138,8 +125,6 @@ def _verify_calls( default_call_list == self._lambda_adapter.return_value.invoke.call_args_list ) - if classifier_adapter_not_called: - self._classifier_adapter.assert_not_called() return _verify_calls @@ -153,27 +138,35 @@ def test_all_latest_plugin_in_dynamo(self, verify_calls): verify_calls() self._zulip.assert_not_called() - @pytest.mark.parametrize("is_plugin_live", [False, True]) def test_stale_plugin_in_dynamo( self, - is_plugin_live, verify_calls, ): self._dynamo_latest_plugins = {PLUGIN: VERSION, "bar": "2.4.6"} self._pypi_latest_plugins = {"bar": "2.4.6"} - self._is_plugin_live = is_plugin_live processor.update_plugin() - if is_plugin_live: - assert len(self._zulip.method_calls) == 0 - put_pm_calls = [] - else: - assert len(self._zulip.method_calls) == 1 - self._zulip.plugin_no_longer_on_hub.assert_called_once_with(PLUGIN) - put_pm_calls = [_create_put_pm_call(PMType.PYPI)] - verify_calls(put_pm_calls=put_pm_calls, classifier_adapter_not_called=False) - self._classifier_adapter.assert_called_once_with(PLUGIN, VERSION) + assert len(self._zulip.method_calls) == 1 + self._zulip.plugin_no_longer_on_hub.assert_called_once_with(PLUGIN) + put_pm_calls = [_create_put_pm_call(PMType.PYPI)] + verify_calls(put_pm_calls=put_pm_calls) + + def test_non_normalized_plugin_name(self, verify_calls): + non_normalized_plugin_name = PLUGIN.upper() + self._dynamo_latest_plugins = { + PLUGIN: VERSION, + non_normalized_plugin_name: VERSION, + } + self._pypi_latest_plugins = {"foo": VERSION} + + processor.update_plugin() + + self._zulip.assert_not_called() + put_pm_calls = [ + _create_put_pm_call(PMType.PYPI, name=non_normalized_plugin_name) + ] + verify_calls(put_pm_calls=put_pm_calls) @pytest.mark.parametrize( "existing_types, put_pm_data, formatted_metadata", @@ -273,8 +266,10 @@ def test_replace_old_plugin_version_with_new( self._zulip.assert_not_called() -def _create_put_pm_call(pm_type, data=None, is_latest=False, version=VERSION) -> call: - kwargs = {"plugin": PLUGIN, "version": version, "plugin_metadata_type": pm_type} +def _create_put_pm_call( + pm_type, data=None, is_latest=False, version=VERSION, name=PLUGIN +) -> call: + kwargs = {"plugin": name, "version": version, "plugin_metadata_type": pm_type} if is_latest: kwargs["is_latest"] = is_latest From 30a5f9dacb93054a27a55c507e3b7ea0d23cc57a Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 21 Jan 2025 10:28:33 -0500 Subject: [PATCH 5/6] Revert "Ensure pypi-normalized names when fetching from npe2api" This reverts commit a7919ecb4f0796659cab0f69a6068bf63d1b2a62. --- data-workflows/plugin/processor.py | 25 ++++++------------- .../tests/utils/test_pypi_adapter.py | 4 +-- .../src/nhcommons/utils/pypi_adapter.py | 13 +--------- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/data-workflows/plugin/processor.py b/data-workflows/plugin/processor.py index 957575706..3f020b575 100644 --- a/data-workflows/plugin/processor.py +++ b/data-workflows/plugin/processor.py @@ -36,7 +36,12 @@ def _is_new_plugin(plugin_version_pair): futures.wait(update_futures, return_when="ALL_COMPLETED") - def _mark_version_stale(name, version) -> None: + # update for removed plugins and existing older version of plugins + for name, version in dynamo_latest_plugins.items(): + pypi_plugin_version = pypi_latest_plugins.get(name) + if pypi_plugin_version == version: + continue + logger.info(f"Updating old plugin={name} version={version}") put_plugin_metadata( plugin=name, @@ -44,22 +49,8 @@ def _mark_version_stale(name, version) -> None: plugin_metadata_type=PluginMetadataType.PYPI, ) - # update for removed plugins and existing older version of plugins - for name, version in dynamo_latest_plugins.items(): - if name != (normalized := pypi_adapter.normalize(name)): - logger.info( - f"Found non-normalized name, plugin={name} will be removed" - f"(normalized={normalized})" - ) - _mark_version_stale(name, version) - continue - - pypi_plugin_version = pypi_latest_plugins.get(name) - if pypi_plugin_version != version: - _mark_version_stale(name, version) - if pypi_plugin_version is None: - logger.info(f"Plugin={name} will be removed from hub") - zulip.plugin_no_longer_on_hub(name) + if pypi_plugin_version is None: + zulip.plugin_no_longer_on_hub(name) def _update_for_new_plugin(name: str, version: str, old_version: Optional[str]) -> None: diff --git a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py index e3ab6712e..dbedcab97 100644 --- a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py @@ -151,9 +151,7 @@ def _mocked_requests_get(self, *args, **kwargs): return MockResponse(status_code=requests.codes.not_found) def test_get_all_plugins(self): - expected = { - pypi_adapter.normalize(plugin[0]): plugin[1] for plugin in plugins() - } + expected = {plugin[0]: plugin[1] for plugin in plugins()} assert expected == pypi_adapter.get_all_plugins() @pytest.mark.parametrize( diff --git a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py index 84b958f87..921b61a00 100644 --- a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py +++ b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py @@ -17,15 +17,6 @@ logger = logging.getLogger(__name__) -def normalize(name: str) -> str: - """ - Normalize the plugin name to lowercase and replace special characters with hyphen. - - https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - """ - return re.sub(r"[-_.]+", "-", name).lower() - - def get_all_plugins() -> Dict[str, str]: """ Query npe2api to get all plugins. @@ -33,15 +24,13 @@ def get_all_plugins() -> Dict[str, str]: Now we use the npe2api to get the list of plugins, which uses the public BigQuery pypi metadata as a source of truth. - This also normalizes the plugins names to match PyPI. - The previous implementation was broken by anti-scraping changes to PyPI. :returns: all plugin names and latest version """ logger.info("Getting all napari plugins from npe2api") packages = get_request(_NPE2API_URL + "/plugins").json() logger.info(f"Total number of napari plugins fetched={len(packages)}") - return {normalize(name): version for name, version in packages.items()} + return packages def get_plugin_pypi_metadata(plugin: str, version: str) -> Dict[str, Any]: From 5b888407a3f55478c55d16ab79fdc34748f88068 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 21 Jan 2025 10:33:45 -0500 Subject: [PATCH 6/6] Remove unnecessary test --- data-workflows/plugin/tests/test_processor.py | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/data-workflows/plugin/tests/test_processor.py b/data-workflows/plugin/tests/test_processor.py index d5044c9e7..ffd266d92 100644 --- a/data-workflows/plugin/tests/test_processor.py +++ b/data-workflows/plugin/tests/test_processor.py @@ -152,22 +152,6 @@ def test_stale_plugin_in_dynamo( put_pm_calls = [_create_put_pm_call(PMType.PYPI)] verify_calls(put_pm_calls=put_pm_calls) - def test_non_normalized_plugin_name(self, verify_calls): - non_normalized_plugin_name = PLUGIN.upper() - self._dynamo_latest_plugins = { - PLUGIN: VERSION, - non_normalized_plugin_name: VERSION, - } - self._pypi_latest_plugins = {"foo": VERSION} - - processor.update_plugin() - - self._zulip.assert_not_called() - put_pm_calls = [ - _create_put_pm_call(PMType.PYPI, name=non_normalized_plugin_name) - ] - verify_calls(put_pm_calls=put_pm_calls) - @pytest.mark.parametrize( "existing_types, put_pm_data, formatted_metadata", [ @@ -266,10 +250,8 @@ def test_replace_old_plugin_version_with_new( self._zulip.assert_not_called() -def _create_put_pm_call( - pm_type, data=None, is_latest=False, version=VERSION, name=PLUGIN -) -> call: - kwargs = {"plugin": name, "version": version, "plugin_metadata_type": pm_type} +def _create_put_pm_call(pm_type, data=None, is_latest=False, version=VERSION) -> call: + kwargs = {"plugin": PLUGIN, "version": version, "plugin_metadata_type": pm_type} if is_latest: kwargs["is_latest"] = is_latest