From 17a2b43a46a0f9dfcbc6f1cb5e4872894e97e34f Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Mon, 22 Jan 2024 15:15:54 -0500 Subject: [PATCH 1/5] Make default port in env file 80 --- .env.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 785719fc3..7bcbb41e1 100644 --- a/.env.sample +++ b/.env.sample @@ -3,7 +3,7 @@ COMPOSE_PROJECT_NAME=cantus #Sets the DEBUG Django setting #PORT = 8000 for development, PORT = 80 for production -PORT=8000 +PORT=80 #Postgres authentication variables POSTGRES_DB=cantus_db From 407ff54b541099a2fbc3a8b632aad0895d101453 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Mon, 22 Jan 2024 15:16:14 -0500 Subject: [PATCH 2/5] Remove migration commands from django-config.sh --- app/django-config.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/django-config.sh b/app/django-config.sh index 9ceb08296..9c65cd058 100644 --- a/app/django-config.sh +++ b/app/django-config.sh @@ -1,7 +1,5 @@ #!/bin/bash -python manage.py makemigrations -python manage.py migrate & python manage.py clear_session_data if [[ $APP_PORT = 80 ]]; then From e2ae2db40f1d7c82bc236c9e4bceba96b50f4930 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Mon, 22 Jan 2024 15:35:28 -0500 Subject: [PATCH 3/5] Update README for committed migrations --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c4d727aad..02cc45d79 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,11 @@ The build process relies on environment variables specified in an `.env` file lo Make a copy of `.env.sample` and name it `.env`. You must make two modifications to this file before the docker containers will build. Both `POSTGRES_PASSWORD` and `RABBIT_PASSWORD` should be uncommented and set with secure passwords. +Before launching the site, ensure that the `PORT` variable is set correctly. For development, it should be set to `8000`; for deployment, it should be `80`. This variable not only sets the port the server listens on, it also configures the application's debug settings. Deploying the website with `PORT=8000` would leak debugging information on a live server and must be avoided. + #### Handling `postgres` authentication issues -When the containers are launched, a volume, `data/postgres`, will be created. When the `POSTGRES_PASSWORD` is changed between builds of the docker containers, the `postgres` container might run into an authentication problem because of the existing volume. If you run into that problem, make sure to remove the volume and re-launch the containers +When the containers are launched, a directory at `./data/postgres`, will be created and mounted to the `postgres` container. When the `POSTGRES_PASSWORD` is changed between builds of the docker containers, the `postgres` container might run into an authentication problem because of the existing volume. If you run into that problem, make sure to remove the volume and re-launch the containers ``` docker rm -v @@ -39,7 +41,7 @@ rm -r data ### Launch in development -In the `.env` file, the `PORT` variable is set to `8000` (development) by default. This will turn on Django's debug mode, showing detailed traces when Django encounters errors, as well as turn off security settings that might stop you from accessing the site locally. In production, this should be set to `80`. +In the `.env` file, the `PORT` variable is set to `8000` (development) by default. This will turn on Django's debug mode, showing detailed traces when Django encounters errors, as well as turn off security settings that might stop you from accessing the site locally. For deployment on a server, this should be set to `80`. > **Windows Users:** Make sure `/app/django-config.sh` has `LF` line endings before launching. This file gets copied over into an Ubuntu container and will break the process if git automatically checked out the file using Windows (`CRLF`) line endings. @@ -47,17 +49,17 @@ Execute the following commands from the root directory of the repo: ```sh # Build the images and launch the containers (this will take a while) -$ docker-compose build -$ docker-compose up -d +$ docker compose build +$ docker compose up -d ``` -When testing your changes, include the `--build` flag to see your changes propagate into the containers: +When testing subsequent changes, you can do this in one step by including the `--build` flag: ``` -docker-compose up --build -d +docker compose up --build -d ``` -After the building process completes (10 to 30 minutes), the site should be available on http://localhost:8000/ in your host machine. +After the build process completes, the site should be available on http://localhost:8000/ in your host machine. By default, Cantus Ultimus works in the following way: @@ -71,17 +73,40 @@ django --> postgres: Use as default database backend django --> solr: Bind django database "signals" to solr ``` -#### Enabling live changes with django's `runserver` -During development, it is often useful to replace `gunicorn` with the default `django` web server, so that modifying the source code results in live changes in the website. This is done by default in `django-config.sh`. If you need to test `gunicorn` during development, edit the if statement in this file. +### Database migrations + +Whenever changes are made to database models, they need to be applied to the PostgreSQL database. This occurs through Django's migrations. + +If, during development, you make a change to any models, build and run the containers, as above. Then, enter the command line in the `app` container: + +```sh +$ docker compose exec -it app bash +``` + +Then, run the `makemigrations` command: + +```sh +$ python manage.py makemigrations +``` + +Migrations will then be created in the `./app/public/cantusdata/migrations/` folder. Commit these migrations and include in the pull request that includes the model changes. + +You will also need to apply these migrations to the database to your development database and, once these model changes are deployed to servers, to staging, production, and any other remote databases. To apply the changes, enter the command line in the `app` container as before and run the `migrate` command: + +```sh +$ python manage.py migrate +``` + +Note that migrations will also need to be applied to a newly instantiated database using the `migrate` command. ### Launch in production -From the Compute Canada VM, follow the same instructions as above, only replace `docker-compose` with `docker compose` and make sure to keep `PORT=80` in the `.env` file. +From the Compute Canada VM, follow the same instructions as above, making sure `PORT=80` is in the `.env` file. -## Initialize a newly launched website +## Initialize a newly launched website (for development or deployment) -A freshly initialized instance of the website does not have an admin account. Addititionally, the databases of Manuscripts, Chants, and Folios are not populated. +A freshly initialized instance of the website does not have an admin account or any data in the database. A few commands will create an admin account and populate the database. @@ -100,13 +125,15 @@ Password (again): Superuser created successfully. ``` -Using your admin credentials, verify that you are able to log into the admin django site, which should be located in http://localhost:8000/admin/ +Using your admin credentials, verify that you are able to log into the admin django site, which should be located at `/admin/`. ![image](https://user-images.githubusercontent.com/7258463/101060564-8be1e500-355d-11eb-887f-5af65b50ba13.png) +When initializing the site, remember to apply migrations to the database (see ) + When navigating through any of the tables in the admin interface (e.g., Manuscripts, and Chants), they will appear to be empty. -We can pre-populate the Manuscripts and Chants from the information available in the [Cantus Database](https://cantusdatabase.org/). +We can populate the Manuscripts and Chants from the information available in the [Cantus Database](https://cantusdatabase.org/). The scripts to populate the database are included in the repository. Head back to the terminal where you created the admin user account. @@ -134,7 +161,8 @@ An additional command is included to import chants associated with a specific ma ```sh $ docker-compose exec app python manage.py import_data chants --manuscript-id MANUSCRIPT_ID ``` -however, this process can already be done using the user interface. We recommend using the user interface from this point onward. + +This process can also be done using the user interface (recommended; see below). ## Adding manuscripts and chants using the admin user interface From 5fbe393a39aac09690c06fd59a345a6a403fdcdb Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Thu, 25 Jan 2024 17:56:14 -0500 Subject: [PATCH 4/5] Add DEVELOPMENT env variable and make default listen port 8000 Uses the DEVELOPMENT environment variable, rather than the PORT variable, to determine whether to run in development or deployment mode. Always listen on port 8000, since nginx does not need to serve over http or https (this is handled by edge routers). --- .env.sample | 6 +++--- README.md | 8 ++++---- app/django-config.sh | 6 +++--- app/public/cantusdata/settings.py | 2 +- docker-compose.yaml | 6 +++--- nginx/nginx.conf | 6 +++--- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.env.sample b/.env.sample index 7bcbb41e1..cd45d9c8d 100644 --- a/.env.sample +++ b/.env.sample @@ -1,9 +1,9 @@ APP_VERSION=3.2-0.11.4 COMPOSE_PROJECT_NAME=cantus -#Sets the DEBUG Django setting -#PORT = 8000 for development, PORT = 80 for production -PORT=80 +#When DEVELOPMENT is False, Django's DEBUG setting +#is False and app is served by gunicorn +DEVELOPMENT=False #Postgres authentication variables POSTGRES_DB=cantus_db diff --git a/README.md b/README.md index 02cc45d79..d2f0ad994 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The build process relies on environment variables specified in an `.env` file lo Make a copy of `.env.sample` and name it `.env`. You must make two modifications to this file before the docker containers will build. Both `POSTGRES_PASSWORD` and `RABBIT_PASSWORD` should be uncommented and set with secure passwords. -Before launching the site, ensure that the `PORT` variable is set correctly. For development, it should be set to `8000`; for deployment, it should be `80`. This variable not only sets the port the server listens on, it also configures the application's debug settings. Deploying the website with `PORT=8000` would leak debugging information on a live server and must be avoided. +Before launching the site, ensure that the `DEVELOPMENT` variable is set correctly. For development, it should be set to `True`; for deployment, it should be `False`. This variable configures the application's debug settings. Deploying the website with `DEVELOPMENT=True` would leak debugging information on a live server and use Django's development server rather than gunicorn and must be avoided. #### Handling `postgres` authentication issues @@ -41,7 +41,7 @@ rm -r data ### Launch in development -In the `.env` file, the `PORT` variable is set to `8000` (development) by default. This will turn on Django's debug mode, showing detailed traces when Django encounters errors, as well as turn off security settings that might stop you from accessing the site locally. For deployment on a server, this should be set to `80`. +In the `.env.sample` file, the `DEVELOPMENT` variable is set to `False` by default. For local development, set this to `True` to turn on Django's debug mode, which allows you to access detailed traces when Django encounters errors. For deployment on a server, this should remain set to `False`. > **Windows Users:** Make sure `/app/django-config.sh` has `LF` line endings before launching. This file gets copied over into an Ubuntu container and will break the process if git automatically checked out the file using Windows (`CRLF`) line endings. @@ -65,7 +65,7 @@ By default, Cantus Ultimus works in the following way: ```mermaid stateDiagram-v2 -[*] --> nginx: Any HTTP request +[*] --> nginx: Any request to port 8000 nginx --> StaticFile: Serve static files directly nginx --> gunicorn: Forward webapp requests to port 8001 gunicorn --> django: Serve content from webapp @@ -102,7 +102,7 @@ Note that migrations will also need to be applied to a newly instantiated databa ### Launch in production -From the Compute Canada VM, follow the same instructions as above, making sure `PORT=80` is in the `.env` file. +From the Compute Canada VM, follow the same instructions as above, making sure `DEVELOPMENT=False` is in the `.env` file. ## Initialize a newly launched website (for development or deployment) diff --git a/app/django-config.sh b/app/django-config.sh index 9c65cd058..ce4dc11a7 100644 --- a/app/django-config.sh +++ b/app/django-config.sh @@ -2,8 +2,8 @@ python manage.py clear_session_data -if [[ $APP_PORT = 80 ]]; then - gunicorn -b 0:8001 cantusdata.wsgi --timeout 600 --workers 4 -else +if [[ $DEVELOPMENT == "True" ]]; then python manage.py runserver_plus 0:8001 +else + gunicorn -b 0:8001 cantusdata.wsgi --timeout 600 --workers 4 fi \ No newline at end of file diff --git a/app/public/cantusdata/settings.py b/app/public/cantusdata/settings.py index ce934f99b..c9f0de0bc 100644 --- a/app/public/cantusdata/settings.py +++ b/app/public/cantusdata/settings.py @@ -11,7 +11,7 @@ from pathlib import Path import os -is_development = os.environ.get("APP_PORT") == "8000" +is_development = os.environ.get("DEVELOPMENT") is_production = not is_development # Build paths inside the project like this: BASE_DIR / 'subdir'. diff --git a/docker-compose.yaml b/docker-compose.yaml index 464ca7f2c..75bc151d6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,7 +13,7 @@ services: condition: service_started environment: - APP_VERSION=${APP_VERSION} - - APP_PORT=${PORT} + - DEVELOPMENT=${DEVELOPMENT} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -41,7 +41,7 @@ services: container_name: cantus-nginx-1 build: ./nginx ports: - - "${PORT}:80" + - "8000:8000" depends_on: - app @@ -86,7 +86,7 @@ services: ] environment: - APP_VERSION=${APP_VERSION} - - APP_PORT=${PORT} + - DEVELOPMENT=${DEVELOPMENT} - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 1b53f903e..33e9e2b58 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -32,18 +32,18 @@ http { server { # if no Host match, close the connection to prevent host spoofing - listen 80 default_server; + listen 8000 default_server; return 444; } server { # use 'listen 80 deferred;' for Linux # use 'listen 80 accept_filter=httpready;' for FreeBSD - listen 80; + listen 8000; client_max_body_size 4G; # set the correct host(s) for your site - server_name cantus.simssa.ca cantus.staging.simssa.ca dev-cantus.simssa.ca localhost + server_name cantus.simssa.ca cantus.staging.simssa.ca dev-cantus.simssa.ca localhost; keepalive_timeout 5; From 15b06d3d8f67056b9e6af1d624d0731d62bfb4d6 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 26 Jan 2024 11:28:31 -0500 Subject: [PATCH 5/5] Fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2f0ad994..96e748f90 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Using your admin credentials, verify that you are able to log into the admin dja ![image](https://user-images.githubusercontent.com/7258463/101060564-8be1e500-355d-11eb-887f-5af65b50ba13.png) -When initializing the site, remember to apply migrations to the database (see ) +When initializing the site, remember to apply migrations to the database (see the "Database Migrations" section above). When navigating through any of the tables in the admin interface (e.g., Manuscripts, and Chants), they will appear to be empty.