From 2bde70c812b7ede62a44036c11dd7113041b31c3 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Wed, 30 Oct 2024 13:24:40 +1030 Subject: [PATCH 1/5] fix: support Meilisearch search engine during setup --- .../management/commands/reindex_course.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/reindex_course.py b/cms/djangoapps/contentstore/management/commands/reindex_course.py index accbc077c4fc..6f4f2c99d8b1 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_course.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_course.py @@ -11,6 +11,7 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator +from search.meilisearch import create_indexes, MeilisearchEngine, INDEX_FILTERABLES from search.search_engine_base import SearchEngine from cms.djangoapps.contentstore.courseware_index import CourseAboutSearchIndexer, CoursewareSearchIndexer @@ -97,14 +98,21 @@ def handle(self, *args, **options): # pylint: disable=too-many-statements logging.exception('Search Engine error - %s', exc) return - index_exists = searcher._es.indices.exists(index=index_name) # pylint: disable=protected-access + # Meilisearch engine + if isinstance(searcher, MeilisearchEngine) and index_name in INDEX_FILTERABLES: + logging.info(f"Creating meilisearch index for {index_name}") + create_indexes({index_name: INDEX_FILTERABLES[index_name]}) - index_mapping = searcher._es.indices.get_mapping( # pylint: disable=protected-access - index=index_name, - ) if index_exists else {} + # Legacy Elasticsearch engine + else: + index_exists = searcher._es.indices.exists(index=index_name) # pylint: disable=protected-access - if index_exists and index_mapping: - return + index_mapping = searcher._es.indices.get_mapping( # pylint: disable=protected-access + index=index_name, + ) if index_exists else {} + + if index_exists and index_mapping: + return # if reindexing is done during devstack setup step, don't prompt the user if setup_option or query_yes_no(self.CONFIRMATION_PROMPT, default="no"): From 7a99d592725c32d4730c0efa9960d59e23def7dc Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Thu, 31 Oct 2024 16:03:42 +1030 Subject: [PATCH 2/5] fix: catch MeilisearchError, but don't call create_indexes Tutor will handle this step for us. --- .../management/commands/reindex_course.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/reindex_course.py b/cms/djangoapps/contentstore/management/commands/reindex_course.py index 6f4f2c99d8b1..d317700e9b2b 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_course.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_course.py @@ -11,7 +11,7 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator -from search.meilisearch import create_indexes, MeilisearchEngine, INDEX_FILTERABLES +from meilisearch.errors import MeilisearchError from search.search_engine_base import SearchEngine from cms.djangoapps.contentstore.courseware_index import CourseAboutSearchIndexer, CoursewareSearchIndexer @@ -94,17 +94,12 @@ def handle(self, *args, **options): # pylint: disable=too-many-statements for index_name in index_names: try: searcher = SearchEngine.get_search_engine(index_name) - except exceptions.ElasticsearchException as exc: + except (exceptions.ElasticsearchException, MeilisearchError) as exc: logging.exception('Search Engine error - %s', exc) return - # Meilisearch engine - if isinstance(searcher, MeilisearchEngine) and index_name in INDEX_FILTERABLES: - logging.info(f"Creating meilisearch index for {index_name}") - create_indexes({index_name: INDEX_FILTERABLES[index_name]}) - # Legacy Elasticsearch engine - else: + if hasattr(searcher, '_es'): # pylint: disable=protected-access index_exists = searcher._es.indices.exists(index=index_name) # pylint: disable=protected-access index_mapping = searcher._es.indices.get_mapping( # pylint: disable=protected-access From fbf89a99019ce0a10fbbf0987855866bdad9da47 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Fri, 1 Nov 2024 11:57:38 +1030 Subject: [PATCH 3/5] fix: remove MeilisearchError catching --- .../contentstore/management/commands/reindex_course.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/reindex_course.py b/cms/djangoapps/contentstore/management/commands/reindex_course.py index d317700e9b2b..8535e925bf3b 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_course.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_course.py @@ -11,7 +11,6 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator -from meilisearch.errors import MeilisearchError from search.search_engine_base import SearchEngine from cms.djangoapps.contentstore.courseware_index import CourseAboutSearchIndexer, CoursewareSearchIndexer @@ -94,7 +93,7 @@ def handle(self, *args, **options): # pylint: disable=too-many-statements for index_name in index_names: try: searcher = SearchEngine.get_search_engine(index_name) - except (exceptions.ElasticsearchException, MeilisearchError) as exc: + except exceptions.ElasticsearchException as exc: logging.exception('Search Engine error - %s', exc) return From a1b3e4445a4defd16d76cb23647f52fe712dd091 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Fri, 1 Nov 2024 12:03:51 +1030 Subject: [PATCH 4/5] fix: bump edx-search requirement --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 88ab7b1c16a5..4836984112af 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -513,7 +513,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/kernel.in # edx-enterprise # edx-proctoring -edx-search==4.1.0 +edx-search==4.1.1 # via -r requirements/edx/kernel.in edx-sga==0.25.0 # via -r requirements/edx/bundled.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 328dfe81381d..2bda2d2adf86 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -806,7 +806,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/testing.txt # edx-enterprise # edx-proctoring -edx-search==4.1.0 +edx-search==4.1.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index fc353debc489..aa8afeaff3fb 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -594,7 +594,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -edx-search==4.1.0 +edx-search==4.1.1 # via -r requirements/edx/base.txt edx-sga==0.25.0 # via -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 0c7d65b95c85..217b9e637350 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -620,7 +620,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -edx-search==4.1.0 +edx-search==4.1.1 # via -r requirements/edx/base.txt edx-sga==0.25.0 # via -r requirements/edx/base.txt From 2e8d7eddc526d821f337421b3d57eded8d52e3d9 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Tue, 5 Nov 2024 13:15:32 +1030 Subject: [PATCH 5/5] fix: allow indexing of empty courses --- cms/djangoapps/contentstore/courseware_index.py | 3 ++- cms/djangoapps/contentstore/tests/test_courseware_index.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/courseware_index.py b/cms/djangoapps/contentstore/courseware_index.py index 48647bf47bc6..d3b6f811d5f6 100644 --- a/cms/djangoapps/contentstore/courseware_index.py +++ b/cms/djangoapps/contentstore/courseware_index.py @@ -256,7 +256,8 @@ def prepare_item_index(item, skip_index=False, groups_usage_info=None): # Now index the content for item in structure.get_children(): prepare_item_index(item, groups_usage_info=groups_usage_info) - searcher.index(items_index, request_timeout=timeout) + if items_index: + searcher.index(items_index, request_timeout=timeout) cls.remove_deleted_items(searcher, structure_key, indexed_items) except Exception as err: # pylint: disable=broad-except # broad exception so that index operation does not prevent the rest of the application from working diff --git a/cms/djangoapps/contentstore/tests/test_courseware_index.py b/cms/djangoapps/contentstore/tests/test_courseware_index.py index 98a60dce901f..3ab3fa373f81 100644 --- a/cms/djangoapps/contentstore/tests/test_courseware_index.py +++ b/cms/djangoapps/contentstore/tests/test_courseware_index.py @@ -504,6 +504,11 @@ def test_delete_course_from_search_index_after_course_deletion(self): """ Test for removing course from CourseAboutSearchIndexer """ self._test_delete_course_from_search_index_after_course_deletion(self.store) + def test_empty_course(self): + empty_course = CourseFactory.create(modulestore=self.store, start=datetime(2015, 3, 1, tzinfo=UTC)) + added_to_index = CoursewareSearchIndexer.do_course_reindex(self.store, empty_course.id) + assert added_to_index == 0 + @patch('django.conf.settings.SEARCH_ENGINE', 'search.tests.utils.ForceRefreshElasticSearchEngine') @ddt.ddt