Merge develop into staging, 13 Aug 2024

.github/workflows/django_tests.yml

@@ -24,6 +24,6 @@ jobs:
                 envkey_AWS_EMAIL_HOST_PASSWORD: test_password
                 directory: config/envs
                 file_name: dev_env
-            - run: docker compose -f docker-compose-development.yml build
-            - run: docker compose -f docker-compose-development.yml up -d
-            - run: docker compose -f docker-compose-development.yml exec -T django python manage.py test main_app.tests
+            - run: docker compose -f docker-compose-test-runner.yml build
+            - run: docker compose -f docker-compose-test-runner.yml up -d
+            - run: docker compose -f docker-compose-test-runner.yml exec -T django python manage.py test main_app.tests

django/cantusdb_project/cantusdb/settings.py

@@ -208,5 +208,6 @@
 
 if DEBUG:
     INSTALLED_APPS.append("debug_toolbar")
+    INSTALLED_APPS.append("django_extensions")
     # debug toolbar must be inserted as early in the middleware as possible
     MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")

django/cantusdb_project/cantusindex.py

@@ -4,7 +4,7 @@
 """
 
 import json
-from typing import Optional, Union, Callable
+from typing import Optional, Union, Callable, TypedDict, Any
 
 import requests
 from requests.exceptions import SSLError, Timeout, HTTPError
@@ -14,122 +14,106 @@
 CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca"
 OLD_CANTUS_INDEX_DOMAIN: str = "https://cantusindex.org"
 DEFAULT_TIMEOUT: float = 2  # seconds
-NUMBER_OF_SUGGESTED_CHANTS: int = 3  # this number can't be too large,
-# since for each suggested chant, we make a request to Cantus Index.
-# We haven't yet parallelized this process, so setting this number
-# too high will cause the Chant Create page to take a very long time
-# to load. If/when we parallelize this process, we want to limit
-# the size of the burst of requests sent to CantusIndex.
+NUMBER_OF_SUGGESTED_CHANTS: int = 5  # default number of suggested chants to return
+# with the get_suggested_chants function
+
+
+class SuggestedChant(TypedDict):
+    """
+    Dictionary containing information required for
+    the suggested chants feature on the Chant Create form.
+    """
+
+    cantus_id: str
+    occurrences: int
+    fulltext: Optional[str]
+    genre_name: Optional[str]
+    genre_id: Optional[int]
 
 
 def get_suggested_chants(
     cantus_id: str, number_of_suggestions: int = NUMBER_OF_SUGGESTED_CHANTS
-) -> Optional[list[dict]]:
+) -> Optional[list[SuggestedChant]]:
+    """
+    Given a Cantus ID, query Cantus Index's /nextchants API for a list of
+    Cantus IDs that follow the given Cantus ID in existing manuscripts.
+    Sort the list by the number of occurrences of each Cantus ID, and return
+    a list of dictionaries containing information about the suggested Cantus IDs
+    with the highest number of occurrences.
+
+    Args:
+        cantus_id (str): a Cantus ID
+        number_of_suggestions (int): the number of suggested Cantus IDs to return
+
+    Returns:
+        Optional[list[dict]]: A list of dictionaries, each containing information
+        about a suggested Cantus ID:
+            - "cantus_id": the suggested Cantus ID
+            - "occurrences": the number of times the suggested Cantus ID follows
+                the given Cantus ID in existing manuscripts
+            - "fulltext": the full text of the suggested Cantus ID
+            - "genre_name": the genre of the suggested Cantus ID
+            - "genre_id": the ID of the genre of the suggested Cantus ID
+            If no suggestions are available, returns None.
+    """
     endpoint_path: str = f"/json-nextchants/{cantus_id}"
-    all_suggestions: Union[list, dict, None] = get_json_from_ci_api(endpoint_path)
+    all_suggestions = get_json_from_ci_api(endpoint_path)
 
-    if not isinstance(all_suggestions, list):
-        # get_json_from_ci_api timed out
-        # or CI returned a response with no suggestions.
+    if all_suggestions is None:
         return None
 
-    # when Cantus ID doesn't exist within CI, CI's api returns a 200 response with `['Cantus ID is not valid']`
+    # when Cantus ID doesn't exist within CI, CI's api returns a
+    # 200 response with `['Cantus ID is not valid']`
     first_suggestion = all_suggestions[0]
     if not isinstance(first_suggestion, dict):
         return None
 
-    sort_by_occurrences: Callable[[dict], int] = lambda suggestion: int(
+    sort_by_occurrences: Callable[[dict[Any, Any]], int] = lambda suggestion: int(
         suggestion["count"]
     )
-    sorted_suggestions: list = sorted(
+    sorted_suggestions: list[dict[Any, Any]] = sorted(
         all_suggestions, key=sort_by_occurrences, reverse=True
     )
-    trimmed_suggestions: list = sorted_suggestions[:number_of_suggestions]
+    trimmed_suggestions = sorted_suggestions[:number_of_suggestions]
 
-    suggested_chants: list[Optional[dict]] = []
+    suggested_chants: list[SuggestedChant] = []
     for suggestion in trimmed_suggestions:
-        cantus_id: str = suggestion["cid"]
-        occurrences: int = int(suggestion["count"])
-        suggested_chants.append(get_suggested_chant(cantus_id, occurrences))
-
-    # filter out Cantus IDs where get_suggested_chant timed out
-    filtered_suggestions: list[dict] = [
-        sugg for sugg in suggested_chants if sugg is not None
-    ]
-
-    return filtered_suggestions
-
-
-def get_suggested_chant(
-    cantus_id: str, occurrences: int, timeout: float = DEFAULT_TIMEOUT
-) -> Optional[dict]:
-    """Given a Cantus ID and a number of occurrences, query one of Cantus Index's
-    APIs for information on that Cantus ID and return a dictionary
-    containing a full text, an incipit, the ID of that Cantus ID's genre, and
-    the number of occurrences for that Cantus ID
-
-    (Number of occurrences: this function is used on the Chant Create page,
-    to suggest Cantus IDs of chants that might follow a chant with the Cantus ID
-    of the most recently created chant within the current source. Number of occurrences
-    is provided by Cantus Index's /nextchants API, based on which chants follow which
-    other chants in existing manuscripts)
-
-    Args:
-        cantus_id (str): a Cantus ID
-        occurrences (int): the number of times chants with this Cantus ID follow chants
-            with the Cantus ID of the most recently created chant.
-
-    Returns:
-        Optional[dict]: A dictionary with the following keys:
-            - "cantus_id"
-            - "occurrences"
-            - "fulltext"
-            - "incipit"
-            - "genre_id"
-            ...but if get_json_from_ci_api timed out, returns None instead
-    """
-    endpoint_path: str = f"/json-cid/{cantus_id}"
-    json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path, timeout=timeout)
-
-    if not isinstance(json, dict):
-        # mostly, in case of a timeout within get_json_from_ci_api
-        return None
+        sugg_cantus_id = suggestion["cid"]
+        occurences = int(suggestion["count"])
+        suggestion_info = suggestion.get("info")
+        if suggestion_info:
+            fulltext = suggestion_info.get("field_full_text")
+            genre_name = suggestion_info.get("field_genre")
+        else:
+            fulltext = None
+            genre_name = None
+        try:
+            genre_id = Genre.objects.get(name=genre_name).id
+        except Genre.DoesNotExist:
+            genre_id = None
+        suggested_chants.append(
+            {
+                "cantus_id": sugg_cantus_id,
+                "occurrences": occurences,
+                "fulltext": fulltext,
+                "genre_name": genre_name,
+                "genre_id": genre_id,
+            }
+        )
 
-    try:
-        fulltext: str = json["info"]["field_full_text"]
-        incipit: str = " ".join(fulltext.split(" ")[:5])
-        genre_name: str = json["info"]["field_genre"]
-    except TypeError:
-        return None
-    genre_id: Optional[int] = None
-    try:
-        genre_id = Genre.objects.get(name=genre_name).id
-    except Genre.DoesNotExist:
-        pass
-
-    clean_cantus_id = cantus_id.replace(".", "d").replace(":", "c")
-    #                                        "d"ot             "c"olon
-    return {
-        "cantus_id": cantus_id,
-        "occurrences": occurrences,
-        "fulltext": fulltext,
-        "incipit": incipit,
-        "genre_name": genre_name,
-        "genre_id": genre_id,
-        "clean_cantus_id": clean_cantus_id,
-    }
+    return suggested_chants
 
 
 def get_suggested_fulltext(cantus_id: str) -> Optional[str]:
     endpoint_path: str = f"/json-cid/{cantus_id}"
-    json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
+    json_response: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
 
-    if not isinstance(json, dict):
+    if not isinstance(json_response, dict):
         # mostly, in case of a timeout within get_json_from_ci_api
         return None
 
     try:
-        suggested_fulltext = json["info"]["field_full_text"]
+        suggested_fulltext = json_response["info"]["field_full_text"]
     except KeyError:
         return None
 
@@ -207,7 +191,7 @@ def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]:
 
 def get_json_from_ci_api(
     path: str, timeout: float = DEFAULT_TIMEOUT
-) -> Union[dict, list, None]:
+) -> Union[dict[Any, Any], list[Any], None]:
     """Given a path, send a request to Cantus Index at that path,
     decode the response to remove its Byte Order Marker, parse it,
     and return it as a dictionary or list.
@@ -221,7 +205,7 @@ def get_json_from_ci_api(
         Union[dict, list, None]:
             If the JSON returned from Cantus Index is a JSON object, returns a dict.
             If the JSON returned is a JSON array, returns a list.
-            In case the request times out, returns None.
+            If the request times out, or other types are returned, returns None.
     """
 
     if not path.startswith("/"):
@@ -243,4 +227,9 @@ def get_json_from_ci_api(
         # there are no suggested chants
         return None
 
-    return response.json()
+    parsed_response = response.json()
+
+    if not isinstance(parsed_response, (dict, list)):
+        return None
+
+    return parsed_response

django/cantusdb_project/main_app/admin/__init__.py

@@ -4,7 +4,7 @@
 from main_app.admin.feast import FeastAdmin
 from main_app.admin.genre import GenreAdmin
 from main_app.admin.notation import NotationAdmin
-from main_app.admin.office import OfficeAdmin
+from main_app.admin.service import ServiceAdmin
 from main_app.admin.provenance import ProvenanceAdmin
 from main_app.admin.segment import SegmentAdmin
 from main_app.admin.sequence import SequenceAdmin

django/cantusdb_project/main_app/admin/chant.py

@@ -1,18 +1,28 @@
 from django.contrib import admin
 
 from main_app.admin.base_admin import EXCLUDE, READ_ONLY, BaseModelAdmin
+from main_app.admin.filters import InputFilter
 from main_app.forms import AdminChantForm
 from main_app.models import Chant
 
 
+class SourceKeyFilter(InputFilter):
+    parameter_name = "source_id"
+    title = "Source ID"
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(source_id=self.value())
+
+
 @admin.register(Chant)
 class ChantAdmin(BaseModelAdmin):
 
     def get_queryset(self, request):
         return (
             super()
             .get_queryset(request)
-            .select_related("source__holding_institution", "genre", "office")
+            .select_related("source__holding_institution", "genre", "service")
         )
 
     @admin.display(description="Source Siglum")
@@ -30,13 +40,15 @@ def get_source_siglum(self, obj):
         "incipit",
         "cantus_id",
         "id",
+        "source__holding_institution__siglum",
     )
 
     readonly_fields = READ_ONLY + ("incipit",)
 
     list_filter = (
+        SourceKeyFilter,
         "genre",
-        "office",
+        "service",
     )
     exclude = EXCLUDE + (
         "col1",

django/cantusdb_project/main_app/admin/sequence.py

@@ -11,7 +11,7 @@ def get_queryset(self, request):
         return (
             super()
             .get_queryset(request)
-            .select_related("source__holding_institution", "genre", "office")
+            .select_related("source__holding_institution", "genre", "service")
         )
 
     @admin.display(description="Source Siglum")
@@ -34,7 +34,7 @@ def get_source_siglum(self, obj):
     list_display = ("incipit", "get_source_siglum", "genre")
     list_filter = (
         "genre",
-        "office",
+        "service",
     )
     raw_id_fields = (
         "source",

django/cantusdb_project/main_app/admin/service.py

@@ -0,0 +1,11 @@
+from django.contrib import admin
+
+from main_app.admin.base_admin import BaseModelAdmin
+from main_app.forms import AdminServiceForm
+from main_app.models import Service
+
+
+@admin.register(Service)
+class ServiceAdmin(BaseModelAdmin):
+    search_fields = ("name",)
+    form = AdminServiceForm

django/cantusdb_project/main_app/admin/source.py

@@ -28,11 +28,15 @@ class SourceAdmin(BaseModelAdmin):
         "id",
         "provenance_notes",
     )
-    readonly_fields = ("title", "siglum") + READ_ONLY + (
-        "number_of_chants",
-        "number_of_melodies",
-        "date_created",
-        "date_updated",
+    readonly_fields = (
+        ("title", "siglum")
+        + READ_ONLY
+        + (
+            "number_of_chants",
+            "number_of_melodies",
+            "date_created",
+            "date_updated",
+        )
     )
     # from the Django docs:
     # Adding a ManyToManyField to this list will instead use a nifty unobtrusive JavaScript “filter” interface