Install and setup Celery workers as a separate container

.dictionary/custom.txt

@@ -109,3 +109,4 @@ xlwt
 xmlfile
 ZIZW
 zustand
+loglevel

backend/app/tasks.py

@@ -0,0 +1,6 @@
+from celery import shared_task
+
+
+@shared_task
+def add(x, y):
+    return x + y

backend/backend/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

backend/backend/celery.py

@@ -0,0 +1,9 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
+
+app = Celery("backend")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks(["app"])

backend/backend/settings.py

@@ -10,12 +10,13 @@
 https://docs.djangoproject.com/en/4.1/ref/settings/
 """
 
-from pathlib import Path
-from django.core.management.utils import get_random_secret_key
 import datetime
+import os
 import sys
+from pathlib import Path
+
+from django.core.management.utils import get_random_secret_key
 from dotenv import load_dotenv
-import os
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -219,15 +220,17 @@
     "JWT_REFRESH_EXPIRATION_DELTA": datetime.timedelta(days=7),
 }
 
+# Configure Celery to use Redis as the broker
+CELERY_BROKER_URL = "redis://message-broker:6379/0"
+CELERY_RESULT_BACKEND = "redis://message-broker:6379/0"
+
 # TODO: Check if we're in DEVELOPMENT_MODE for localhost vs channel service
 # Channel layer configurations
 CHANNEL_LAYERS = {
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
         "CONFIG": {
-            "hosts": [
-                (os.environ.get("REDIS_CHANNELS_HOST", "localhost"), 6379)
-            ],  # Configure Redis server
+            "hosts": [(os.environ.get("REDIS_CHANNELS_HOST", "localhost"), 6379)],  # Configure Redis server
         },
     },
 }

backend/docker-compose.dev.yml

@@ -3,22 +3,34 @@ version: "3.9"
 services:
   web:
     build: .
-    container_name: "backend_DEV"
+    container_name: "crumpet-backend-dev"
     command: python manage.py runserver 0.0.0.0:8000
     volumes:
       - .:/code
-      # - ./app:/code/app # For live reload
     ports:
       - "8000:8000"
     tty: true
     env_file:
       - .env.development
     depends_on:
       - db
+      - message-broker
+
+  celery:
+    build: .
+    container_name: "crumpet-celery-worker-dev"
+    command: celery -A backend worker --loglevel=info
+    volumes:
+      - .:/code
+    env_file:
+      - .env.development
+    depends_on:
+      - web
+      - message-broker
 
   db:
     image: postgres:15
-    container_name: "crumpet_postgres_DEV"
+    container_name: "crumpet-postgres-dev"
     ports:
       - "5432:5432"
     volumes:
@@ -30,6 +42,12 @@ services:
       - POSTGRES_USER=${DB_USERNAME}
       - POSTGRES_PASSWORD=${DB_PASSWORD}
 
+  message-broker:
+    image: "redis:latest"
+    container_name: "crumpet-redis-message-broker-dev"
+    ports:
+      - "6379:6379"
+
   # redis_channels:
   #   image: "redis:latest"
   #   container_name: "crumpet_redis_channels_DEV"
@@ -40,6 +58,6 @@ services:
 
 volumes:
   pgdata:
-  # redis_channels_data:
+    # redis_channels_data:
 
 

backend/requirements.txt

@@ -1,15 +1,21 @@
+amqp==5.2.0
 anyio==4.0.0
 asgiref==3.6.0
 astroid==2.15.6
 async-timeout==4.0.3
 attrs==23.1.0
+billiard==4.2.0
 black==24.1.1
+celery==5.3.6
 certifi==2022.12.7
 cfgv==3.3.1
 channels==4.0.0
 channels-redis==4.1.0
 charset-normalizer==3.0.1
 click==8.1.7
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.3.0
 coreapi==2.3.3
 coreschema==0.0.4
 dill==0.3.6
@@ -39,6 +45,7 @@ itypes==1.2.0
 Jinja2==3.1.2
 jsonschema==4.20.0
 jsonschema-specifications==2023.11.2
+kombu==5.3.5
 lazy-object-proxy==1.9.0
 MarkupSafe==2.1.2
 mccabe==0.7.0
@@ -52,6 +59,7 @@ packaging==23.2
 pathspec==0.12.1
 platformdirs==3.0.0
 pre-commit==3.0.4
+prompt-toolkit==3.0.43
 psycopg2==2.9.7
 psycopg2-binary==2.9.5
 pycodestyle==2.10.0
@@ -60,6 +68,7 @@ pycryptodome-test-vectors==1.0.13
 pyflakes==3.0.1
 PyJWT==1.7.1
 pylint==2.17.4
+python-dateutil==2.9.0
 python-dotenv==0.21.1
 pytz==2022.7.1
 PyYAML==6.0
@@ -68,6 +77,7 @@ referencing==0.32.0
 requests==2.28.2
 rpds-py==0.13.2
 simplejson==3.18.3
+six==1.16.0
 sniffio==1.3.0
 sqlparse==0.4.3
 tomli==2.0.1
@@ -77,12 +87,15 @@ types-PyYAML==6.0.12.12
 types-requests==2.31.0.6
 types-urllib3==1.26.25.14
 typing_extensions==4.7.1
+tzdata==2024.1
 uritemplate==4.1.1
 urllib3==1.26.14
 uvicorn==0.24.0.post1
 uvloop==0.19.0
+vine==5.1.0
 virtualenv==20.19.0
 watchfiles==0.21.0
+wcwidth==0.2.13
 websockets==12.0
 wrapt==1.15.0
 xlrd==2.0.1

cspell.config.yaml

@@ -8,3 +8,5 @@ ignorePaths:
   - "**/migrations/**"
   - "**/node_modules/**"
   - "**/.vscode/**"
+  - "**/.git/**"
+  - "**/requirements.txt"