Caching

.gitignore

@@ -31,7 +31,7 @@ __pycache__
 */django_logger.log
 **/secrets.py
 **.pyc
-
+coptic/cache/
 # --------------------
 # Env Files
 # --------------------

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Remote Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/var/www/cts"
+                }
+            ],
+            "justMyCode": true
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,17 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Docker Compose Up",
+            "type": "shell",
+            "command": "docker-compose up --build",
+            "problemMatcher": []
+        },
+        {
+            "label": "Docker Compose Down",
+            "type": "shell",
+            "command": "docker-compose down",
+            "problemMatcher": []
+        }
+    ]
+}

coptic/.dockerignore

@@ -0,0 +1,27 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

coptic/Dockerfile

@@ -0,0 +1,42 @@
+# Base image
+FROM ubuntu:20.04
+
+# Environment settings
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install key packages
+RUN apt-get update && apt-get install -y \
+    apache2 \
+    apache2-dev \
+    chromium-browser \
+    git \
+    libapache2-mod-wsgi-py3 \
+    default-libmysqlclient-dev \
+    build-essential \
+    pkg-config \
+    python3-pip \
+    redis-server \
+    redis-tools \
+    unzip \
+    xvfb \
+    && apt-get clean
+
+# Install Python dependencies
+COPY coptic/requirements.txt /var/www/cts/coptic/requirements.txt
+RUN pip3 install -r /var/www/cts/coptic/requirements.txt
+RUN pip3 install debugpy
+
+# Copy application files
+WORKDIR /var/www/cts/coptic
+COPY ./coptic /var/www/cts/coptic
+RUN mkdir -p /var/www/cts/coptic/scripts
+COPY ./scripts/ /var/www/cts/coptic/scripts
+
+# Apache configuration
+COPY ./ansible/roles/scriptorium/files/scriptorium.conf /etc/apache2/sites-available/000-default.conf
+
+EXPOSE 80
+# Environment variables for Django
+ENV DJANGO_SETTINGS_MODULE="coptic.settings.docker"
+# Command to start Apache
+CMD ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "manage.py", "runserver", "0.0.0.0:8000"]

coptic/README_Docker.md

@@ -0,0 +1,9 @@
+# Developing with visual studio code
+
+```
+export SECRET_KEY="what you want"
+export GITHUB_TOKEN="what you need"
+docker compose build
+docker compose up
+code .
+```

coptic/coptic/settings/docker.py

@@ -0,0 +1,86 @@
+import os
+from pathlib import Path
+SECRET_KEY="what you want" # this is local development only
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
+
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+# Application definition
+INSTALLED_APPS = (
+    'grappelli',
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'texts',
+    #'annis',
+    #'ingest',
+    'api',
+    'mod_wsgi.server'
+)
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+]
+
+ROOT_URLCONF = 'coptic.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [os.path.join(BASE_DIR, 'templates')],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'coptic.wsgi.application'
+
+# Database
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': os.environ.get('MYSQL_DB', 'coptic'),
+        'USER': os.environ.get('MYSQL_USER', 'coptic'),
+        'PASSWORD': os.environ.get('MYSQL_PASSWORD', ''),
+        'HOST': os.environ.get('MYSQL_HOST', 'db'),
+        'PORT': os.environ.get('MYSQL_PORT', '3306'),
+    }
+}
+# Cache configuration
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': os.path.join(BASE_DIR, 'cache'),
+    }
+}
+# Static files (CSS, JavaScript, Images)
+STATIC_URL = '/static/'
+STATIC_ROOT = "/var/www/cts/coptic/static/"
+

coptic/coptic/views.py

@@ -5,6 +5,8 @@
 from django.db.models import Q, Case, When, IntegerField, F
 from django.shortcuts import render, get_object_or_404, redirect
 from django.db.models.functions import Lower
+from django.views.decorators.cache import cache_page
+from django.core.cache import cache
 from texts.search_fields import get_search_fields
 from coptic.settings.base import DEPRECATED_URNS
 from collections import OrderedDict
@@ -19,12 +21,13 @@
 def keyvalue(dict, key):
     return dict.get(key)
 
+@cache_page(60 * 15)
 def home_view(request):
     'Home'
     context = _base_context()
     return render(request, 'home.html', context)
 
-
+@cache_page(60 * 15)
 def corpus_view(request, corpus=None):
     corpus_object = get_object_or_404(models.Corpus, slug=corpus)
 
@@ -64,7 +67,7 @@ def corpus_view(request, corpus=None):
     })
     return render(request, 'corpus.html', context)
 
-
+@cache_page(60 * 15)
 def text_view(request, corpus=None, text=None, format=None):
     text_object = get_object_or_404(models.Text, slug=text)
     if not format:
@@ -107,11 +110,10 @@ def text_view(request, corpus=None, text=None, format=None):
     })
     return render(request, 'text.html', context)
 
-
+@cache_page(60 * 15)
 def not_found(request):
     return render(request, '404.html', {})
 
-
 def _resolve_urn(urn):
     try:
         text = models.Text.objects.get(text_meta__name="document_cts_urn", text_meta__value=urn)
@@ -123,7 +125,7 @@ def _resolve_urn(urn):
         except models.Corpus.DoesNotExist:
             return None
 
-
+@cache_page(60 * 15)
 def urn(request, urn=None):
     # https://github.com/CopticScriptorium/cts/issues/112
     if re.match(r'urn:cts:copticLit:ot.*.crosswire', urn):
@@ -139,7 +141,6 @@ def urn(request, urn=None):
         return redirect('corpus', corpus=obj.slug)
     return redirect(reverse('search') + f"?text={urn}")
 
-
 def get_meta_values(meta):
     unsplit_values = map(lambda x: x['value'], models.TextMeta.objects.filter(name__iexact=meta.name).values("value").distinct())
     if not meta.splittable:
@@ -157,7 +158,7 @@ def get_meta_values(meta):
     meta_values = [re.sub(HTML_TAG_REGEX, '', meta_value) for meta_value in meta_values]
     return meta_values
 
-
+@cache_page(60 * 15)
 def index_view(request, special_meta=None):
     context = _base_context()
 
@@ -299,7 +300,6 @@ def _fetch_and_filter_texts_for_special_metadata_query(queries):
     add_author_and_urn(texts)
     return texts
 
-
 def _build_explanation(params):
     meta_explanations = []
     for meta_name, meta_values in params.items():
@@ -326,7 +326,6 @@ def _build_explanation(params):
         meta_explanations.append("(" + " OR ".join(meta_name_explanations) + ")")
     return " AND ".join(meta_explanations)
 
-
 def _build_result_for_query_text(params, texts, explanation):
     query_text = params["text"]
     results = []
@@ -351,16 +350,18 @@ def _build_result_for_query_text(params, texts, explanation):
     all_empty_explanation += explanation
     return results, all_empty_explanation
 
-
 def _base_context():
-    search_fields = get_search_fields()
-    context = {
-        'search_fields': search_fields[:5],
-        'secondary_search_fields': search_fields[5:]
-    }
+    context = cache.get('base_context')
+    if not context:
+        search_fields = get_search_fields()
+        context = {
+            'search_fields': search_fields[:5],
+            'secondary_search_fields': search_fields[5:]
+        }
+        cache.set('base_context', context, 60 * 15)  # Cache for 15 minutes
     return context
 
-
+@cache_page(60 * 15)
 def search(request):
     context = _base_context()
 
@@ -418,7 +419,6 @@ def search(request):
 
     return render(request, 'search.html', context)
 
-
 def add_author_and_urn(texts):
     for text in texts:
         try:

coptic/requirements.txt

@@ -1,10 +1,10 @@
 requests
-django
-django-grappelli
-beautifulsoup4
-selenium==2.45.0
-xvfbwrapper==0.2.4
-mod-wsgi==4.4.11
-celery==3.1.18
+django==2.2.13
+mysqlclient
+django-grappelli==2.13.1
+beautifulsoup4==4.8.0
+selenium==3.141.0
+xvfbwrapper==0.2.9
+mod-wsgi
+celery
 github3.py==1.3.0
-tqdm