From 4a0364305faa8192133e953ee485c01191bb4944 Mon Sep 17 00:00:00 2001 From: tomtitherington Date: Fri, 1 Mar 2024 17:30:22 +0000 Subject: [PATCH] Install and setup Celery workers as a separate container --- .dictionary/custom.txt | 1 + backend/app/tasks.py | 6 ++++++ backend/backend/__init__.py | 3 +++ backend/backend/celery.py | 9 +++++++++ backend/backend/settings.py | 15 +++++++++------ backend/docker-compose.dev.yml | 26 ++++++++++++++++++++++---- backend/requirements.txt | 13 +++++++++++++ cspell.config.yaml | 2 ++ 8 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 backend/app/tasks.py create mode 100644 backend/backend/celery.py diff --git a/.dictionary/custom.txt b/.dictionary/custom.txt index 1162db1..a9b90a4 100644 --- a/.dictionary/custom.txt +++ b/.dictionary/custom.txt @@ -109,3 +109,4 @@ xlwt xmlfile ZIZW zustand +loglevel diff --git a/backend/app/tasks.py b/backend/app/tasks.py new file mode 100644 index 0000000..8335010 --- /dev/null +++ b/backend/app/tasks.py @@ -0,0 +1,6 @@ +from celery import shared_task + + +@shared_task +def add(x, y): + return x + y diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py index e69de29..53f4ccb 100644 --- a/backend/backend/__init__.py +++ b/backend/backend/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/backend/backend/celery.py b/backend/backend/celery.py new file mode 100644 index 0000000..b6d7706 --- /dev/null +++ b/backend/backend/celery.py @@ -0,0 +1,9 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") + +app = Celery("backend") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks(["app"]) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index ffa0684..da6b6ad 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -10,12 +10,13 @@ https://docs.djangoproject.com/en/4.1/ref/settings/ """ -from pathlib import Path -from django.core.management.utils import get_random_secret_key import datetime +import os import sys +from pathlib import Path + +from django.core.management.utils import get_random_secret_key from dotenv import load_dotenv -import os # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -219,15 +220,17 @@ "JWT_REFRESH_EXPIRATION_DELTA": datetime.timedelta(days=7), } +# Configure Celery to use Redis as the broker +CELERY_BROKER_URL = "redis://message-broker:6379/0" +CELERY_RESULT_BACKEND = "redis://message-broker:6379/0" + # TODO: Check if we're in DEVELOPMENT_MODE for localhost vs channel service # Channel layer configurations CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [ - (os.environ.get("REDIS_CHANNELS_HOST", "localhost"), 6379) - ], # Configure Redis server + "hosts": [(os.environ.get("REDIS_CHANNELS_HOST", "localhost"), 6379)], # Configure Redis server }, }, } diff --git a/backend/docker-compose.dev.yml b/backend/docker-compose.dev.yml index 2b8f035..c330542 100644 --- a/backend/docker-compose.dev.yml +++ b/backend/docker-compose.dev.yml @@ -3,11 +3,10 @@ version: "3.9" services: web: build: . - container_name: "backend_DEV" + container_name: "crumpet-backend-dev" command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/code - # - ./app:/code/app # For live reload ports: - "8000:8000" tty: true @@ -15,10 +14,23 @@ services: - .env.development depends_on: - db + - message-broker + + celery: + build: . + container_name: "crumpet-celery-worker-dev" + command: celery -A backend worker --loglevel=info + volumes: + - .:/code + env_file: + - .env.development + depends_on: + - web + - message-broker db: image: postgres:15 - container_name: "crumpet_postgres_DEV" + container_name: "crumpet-postgres-dev" ports: - "5432:5432" volumes: @@ -30,6 +42,12 @@ services: - POSTGRES_USER=${DB_USERNAME} - POSTGRES_PASSWORD=${DB_PASSWORD} + message-broker: + image: "redis:latest" + container_name: "crumpet-redis-message-broker-dev" + ports: + - "6379:6379" + # redis_channels: # image: "redis:latest" # container_name: "crumpet_redis_channels_DEV" @@ -40,6 +58,6 @@ services: volumes: pgdata: - # redis_channels_data: + # redis_channels_data: diff --git a/backend/requirements.txt b/backend/requirements.txt index ef58366..607c22d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,15 +1,21 @@ +amqp==5.2.0 anyio==4.0.0 asgiref==3.6.0 astroid==2.15.6 async-timeout==4.0.3 attrs==23.1.0 +billiard==4.2.0 black==24.1.1 +celery==5.3.6 certifi==2022.12.7 cfgv==3.3.1 channels==4.0.0 channels-redis==4.1.0 charset-normalizer==3.0.1 click==8.1.7 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 coreapi==2.3.3 coreschema==0.0.4 dill==0.3.6 @@ -39,6 +45,7 @@ itypes==1.2.0 Jinja2==3.1.2 jsonschema==4.20.0 jsonschema-specifications==2023.11.2 +kombu==5.3.5 lazy-object-proxy==1.9.0 MarkupSafe==2.1.2 mccabe==0.7.0 @@ -52,6 +59,7 @@ packaging==23.2 pathspec==0.12.1 platformdirs==3.0.0 pre-commit==3.0.4 +prompt-toolkit==3.0.43 psycopg2==2.9.7 psycopg2-binary==2.9.5 pycodestyle==2.10.0 @@ -60,6 +68,7 @@ pycryptodome-test-vectors==1.0.13 pyflakes==3.0.1 PyJWT==1.7.1 pylint==2.17.4 +python-dateutil==2.9.0 python-dotenv==0.21.1 pytz==2022.7.1 PyYAML==6.0 @@ -68,6 +77,7 @@ referencing==0.32.0 requests==2.28.2 rpds-py==0.13.2 simplejson==3.18.3 +six==1.16.0 sniffio==1.3.0 sqlparse==0.4.3 tomli==2.0.1 @@ -77,12 +87,15 @@ types-PyYAML==6.0.12.12 types-requests==2.31.0.6 types-urllib3==1.26.25.14 typing_extensions==4.7.1 +tzdata==2024.1 uritemplate==4.1.1 urllib3==1.26.14 uvicorn==0.24.0.post1 uvloop==0.19.0 +vine==5.1.0 virtualenv==20.19.0 watchfiles==0.21.0 +wcwidth==0.2.13 websockets==12.0 wrapt==1.15.0 xlrd==2.0.1 diff --git a/cspell.config.yaml b/cspell.config.yaml index 870b70f..e9b04cf 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -8,3 +8,5 @@ ignorePaths: - "**/migrations/**" - "**/node_modules/**" - "**/.vscode/**" + - "**/.git/**" + - "**/requirements.txt"