Merge pull request #537 from ucb-rit/develop

v3.4.0 Changes

.env

@@ -0,0 +1,3 @@
+# Environment variables used by Docker (specifically for docker-compose.yml)
+DB_NAME=cf_brc_db
+COLDFRONT_PORT=8880

.github/workflows/django_testing_ci.yml

@@ -84,6 +84,7 @@ jobs:
     - name: Run Tests
       run: |
         source ~/venv/bin/activate
+        export django_secret_key=`openssl rand -base64 64`
         python manage.py migrate
         python manage.py test
 

Dockerfile

@@ -1,34 +1,33 @@
-FROM centos:8
+FROM centos/python-38-centos7
 
 LABEL description="coldfront"
 
-# install dependencies
-RUN yum -y install epel-release
-RUN yum -y update
-RUN yum -y install python36 python36-devel git memcached redis
-
+USER root
 WORKDIR /root
-
-# install coldfront
-RUN mkdir /opt/coldfront_app
-
-WORKDIR /opt/coldfront_app
-
-RUN cd /opt/coldfront_app
-RUN git clone https://github.com/ubccr/coldfront.git
-RUN python3.6 -mvenv venv
-RUN source venv/bin/activate
-
-WORKDIR /opt/coldfront_app/coldfront
-
-RUN cd /opt/coldfront_app/coldfront
-RUN pip3 install wheel
-RUN pip3 install -r requirements.txt
-RUN cp coldfront/config/local_settings.py.sample coldfront/config/local_settings.py
-RUN cp coldfront/config/local_strings.py.sample coldfront/config/local_strings.py
-
-RUN python3 ./manage.py initial_setup
-RUN python3 ./manage.py load_test_data
-
-EXPOSE 8000
+COPY requirements.txt ./
+RUN pip install -r requirements.txt && rm requirements.txt
+RUN pip install jinja2 pyyaml
+
+# mybrc or mylrc
+ARG PORTAL="mybrc"
+RUN mkdir -p /var/log/user_portals/cf_${PORTAL} \
+ && touch /var/log/user_portals/cf_${PORTAL}/cf_${PORTAL}_{portal,api}.log \
+ && chmod 775 /var/log/user_portals/cf_${PORTAL} \
+ && chmod 664 /var/log/user_portals/cf_${PORTAL}/cf_${PORTAL}_{portal,api}.log
+
+COPY . /vagrant/coldfront_app/coldfront/
+WORKDIR /vagrant/coldfront_app/coldfront/
+
+RUN chmod +x ./manage.py
+
+CMD ./manage.py initial_setup \
+ && ./manage.py add_accounting_defaults \
+ && ./manage.py add_allowance_defaults \
+ && ./manage.py add_directory_defaults \
+ && ./manage.py create_allocation_periods \
+ && ./manage.py create_staff_group \
+ && ./manage.py collectstatic --noinput \
+ && ./manage.py runserver 0.0.0.0:80
+
+EXPOSE 80
 STOPSIGNAL SIGINT

Dockerfile.db

@@ -0,0 +1,5 @@
+FROM postgres:15
+
+LABEL description="coldfront db"
+
+RUN apt update && apt install -y sudo postgresql-client

README.md

@@ -46,28 +46,34 @@ of variables used by Ansible to configure the system.
    ```
    cp bootstrap/ansible/main.copyme main.yml
    ```
-7. Customize `main.yml`. In particular, uncomment everything under the `dev_settings` section, and fill in the below variables. Note
+7. Generate a key to be used as the `SECRET_KEY` for Django.
+   ```
+   # This produces two lines: condense them into one.
+   openssl rand -base64 64
+   ```
+8. Customize `main.yml`. In particular, uncomment everything under the `dev_settings` section, and fill in the below variables. Note
 that quotes need not be provided, except in the list variable.
    ```
+   django_secret_key: secret_key_from_previous_step
    db_admin_passwd: password_here
    redis_passwd: password_here
    from_email: [email protected]
    admin_email: [email protected]
    email_admin_list: ["[email protected]"]
    request_approval_cc_list: ["[email protected]"]
    ```
-8. Provision the VM. This should run the Ansible playbook. Expect this to take
+9. Provision the VM. This should run the Ansible playbook. Expect this to take
 a few minutes on the first run.
    ```
    vagrant up
    ```
-9. SSH into the VM.
+10. SSH into the VM.
    ```
    vagrant ssh
    ```
-10. On the host machine, navigate to `http://localhost:8880`, where the
+11. On the host machine, navigate to `http://localhost:8880`, where the
 application should be served.
-11. (Optional) Load data from a database dump file.
+12. (Optional) Load data from a database dump file.
     ```
     # Clear the Django database to avoid conflicts.
     python manage.py sqlflush | python manage.py dbshell
@@ -207,6 +213,15 @@ multiple files or directories to omit.
     - Open `htmlcov/index.html` in a browser to view which lines of 
   code were covered by the tests and which were not.
 
+## Docker - Quick Install (Recommend)
+1. Generate configuration (`dev_settings.py`): have Python with the `jinja2` and `pyyaml` libraries installed, and then run `bootstrap/development/gen_config.sh`
+2. Build Images: In the base directory, run `docker build . -t coldfront` and `docker build . -f Dockerfile.db -t coldfront_db`. `--build-arg PORTAL=mylrc` can be added to the build command to build for mylrc.
+3. If needed, modify `.env` to customize the web server port and the database name (e.g from `cf_mybrc_db` to `cf_mylrc_db`)
+4. To run: In the base directory, run `docker compose up`
+5. To enter the coldfront container (similar to `vagrant ssh`): run `docker exec -it coldfront-coldfront-1 bash`
+6. To load a database backup: run `bootstrap/development/docker_load_database_backup.sh ${DB_NAME} ${PATH_TO_DUMP}`
+7. To start from scratch (delete volumes): In the base directory, run `docker compose down --volumes`
+
 ## Local Machine - Quick Install (Not Recommended)
 
 1. ColdFront requires Python 3.6, memcached, and redis.

bootstrap/ansible/main.copyme

@@ -18,9 +18,14 @@ djangoprojname: coldfront
 # BRC/LRC Config
 ###############################################################################
 
+# The secret key used by Django for cryptographic signing.
+# TODO: Generate one using: `openssl rand -base64 64`.
+django_secret_key:
+
 # The name of the PostgreSQL database.
 # TODO: For LRC, set this to 'cf_lrc_db'.
 db_name: cf_brc_db
+db_host: localhost
 
 # The credentials for the database admin user.
 # TODO: Replace the username and password.
@@ -30,6 +35,7 @@ db_admin_passwd: ''
 # The password for Redis.
 # TODO: Replace the password.
 redis_passwd: ''
+redis_host: localhost
 
 # Log paths.
 # TODO: For LRC, use the substring 'cf_mylrc'.
@@ -81,16 +87,18 @@ cilogon_app_client_id: ""
 cilogon_app_secret: ""
 
 # Django Flags settings.
-# TODO: For LRC, disable basic auth. and enable SSO.
-flag_basic_auth_enabled: True
-flag_sso_enabled: False
+# TODO: For LRC, disable link login.
+flag_basic_auth_enabled: False
+flag_sso_enabled: True
+flag_link_login_enabled: True
 # TODO: For LRC, disable BRC and enable LRC.
 flag_brc_enabled: True
 flag_lrc_enabled: False
 # The number of the month in which users should be able to request renewal for
 # the next allowance year.
 # TODO: For LRC, set the month number to 9 (September).
 flag_next_period_renewal_requestable_month: 5
+flag_multiple_email_addresses_allowed: False
 
 # Portal settings.
 # TODO: For LRC, use "MyLRC", "Laboratory Research Computing", "LRC", and

bootstrap/ansible/settings_template.tmpl

@@ -3,6 +3,8 @@ import os
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = {{ debug }}
 
+SECRET_KEY = '{{ django_secret_key }}'
+
 ALLOWED_HOSTS = ['0.0.0.0', '{{ hostname }}', '{{ host_ip }}']
 
 PORTAL_NAME = '{{ portal_name }}'
@@ -46,7 +48,7 @@ DATABASES = {
         'NAME': '{{ db_name }}',
         'USER': '{{ db_admin_user }}',
         'PASSWORD': '{{ db_admin_passwd }}',
-        'HOST': 'localhost',
+        'HOST': '{{ db_host }}',
         'PORT': '5432',
     },
 }
@@ -108,7 +110,8 @@ if SENTRY_DSN:
     sentry_sdk.init(
         dsn=SENTRY_DSN,
         integrations=[DjangoIntegration()],
-        traces_sample_rate=0.01,
+        sample_rate=0.1,
+        traces_sample_rate=0.001,
         send_default_pii=True)
     # Ignore noisy loggers.
     ignore_logger('coldfront.api')
@@ -135,7 +138,7 @@ CONSTANCE_CONFIG = {
     'LAUNCH_DATE': (date(1970, 1, 1), 'The date the portal was launched.'),
 }
 CONSTANCE_REDIS_CONNECTION = {
-    'host': '127.0.0.1',
+    'host': '{{ redis_host }}',
     'port': 6379,
     'db': 0,
     'password': '{{ redis_passwd }}',
@@ -152,6 +155,12 @@ EXTRA_INTERNAL_IPS = [
 {% endfor %}
 ]
 
+if DEBUG:
+    # For Docker support
+    import socket
+    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind('.')] + '.1' for ip in ips] + ['10.0.2.2']
+
 #------------------------------------------------------------------------------
 # django-flags settings
 #------------------------------------------------------------------------------
@@ -162,19 +171,37 @@ FLAGS = {
     ],
     'BASIC_AUTH_ENABLED': [{'condition': 'boolean', 'value': {{ flag_basic_auth_enabled }}}],
     'BRC_ONLY': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}],
+    'LINK_LOGIN_ENABLED': [{'condition': 'boolean', 'value': {{ flag_link_login_enabled }}}],
     'LRC_ONLY': [{'condition': 'boolean', 'value': {{ flag_lrc_enabled }}}],
+    'MULTIPLE_EMAIL_ADDRESSES_ALLOWED': [{'condition': 'boolean', 'value': {{ flag_multiple_email_addresses_allowed }}}],
     'SECURE_DIRS_REQUESTABLE': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}],
     'SERVICE_UNITS_PURCHASABLE': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}],
     'SSO_ENABLED': [{'condition': 'boolean', 'value': {{ flag_sso_enabled }}}],
 }
 
+# Enforce that boolean flags are consistent with each other.
+from django.core.exceptions import ImproperlyConfigured
+if not (FLAGS['BRC_ONLY'][0]['value'] ^ FLAGS['LRC_ONLY'][0]['value']):
+    raise ImproperlyConfigured(
+        'Exactly one of BRC_ONLY, LRC_ONLY should be enabled.')
+if not (
+        FLAGS['BASIC_AUTH_ENABLED'][0]['value'] ^
+        FLAGS['SSO_ENABLED'][0]['value']):
+    raise ImproperlyConfigured(
+        'Exactly one of BASIC_AUTH_ENABLED, SSO_ENABLED should be enabled.')
+if (not FLAGS['SSO_ENABLED'][0]['value'] and
+        FLAGS['LINK_LOGIN_ENABLED'][0]['value']):
+    raise ImproperlyConfigured(
+        'LINK_LOGIN_ENABLED should only be enabled when SSO_ENABLED is '
+        'enabled.')
+
 #------------------------------------------------------------------------------
 # django-q settings
 #------------------------------------------------------------------------------
 
 Q_CLUSTER = {
     'redis': {
-        'host': '127.0.0.1',
+        'host': '{{ redis_host }}',
         'port': 6379,
         'db': 0,
         'password': '{{ redis_passwd }}',

bootstrap/development/docker_load_database_backup.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# $1 = database name
+# $2 = dump file
+docker exec -i coldfront-db-1 pg_restore --verbose --clean -U admin -d $1 < $2

bootstrap/development/gen_config.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+cp coldfront/config/local_settings.py.sample \
+    coldfront/config/local_settings.py
+cp coldfront/config/local_strings.py.sample \
+    coldfront/config/local_strings.py
+python -c \
+"from jinja2 import Template, Environment, FileSystemLoader; \
+import yaml; \
+env = Environment(loader=FileSystemLoader('bootstrap/ansible/')); \
+env.filters['bool'] = lambda x: str(x).lower() in ['true', 'yes', 'on', '1']; \
+options = yaml.safe_load(open('main.yml').read()); \
+options.update({'redis_host': 'redis', 'db_host': 'db'}); \
+print(env.get_template('settings_template.tmpl').render(options))" \
+                                              > coldfront/config/dev_settings.py

bootstrap/development/load_database_backup.sh

@@ -1,18 +1,27 @@
-#!/bin/sh
+#!/bin/bash
 
 # This script clears out the PostgreSQL database with the given name and
 # loads in the data from the given dump file generated by pg_dump. It
 # skips loading entries from the Job table.
 
 # Store second last and last arguments.
-DB_NAME=${@:(-2):1}
+if [ "$POSTGRES_DB" != "" ]; then
+    DB_NAME=$POSTGRES_DB
+    if [ "$LOAD_DUMP" = true ]; then
+        DUMP_FILE=/db.dump
+    else
+        exit 0
+    fi
+else
+    DB_NAME=${@:(-2):1}
+    DUMP_FILE=${@: -1}
+fi
 DB_OWNER=admin
-DUMP_FILE=${@: -1}
 
 BIN=/usr/pgsql-15/bin/pg_restore
 
 if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] ; then
-    echo "Usage: `basename $0` [OPTION] name-of-database absolute-path-to-dump-file"
+    echo "Usage: sudo -u postgres `basename $0` [OPTION] name-of-database absolute-path-to-dump-file"
     echo $'  -k, --kill-connections\t\tterminates all connections to the database so it can be dropped'
     exit 0
 elif ! [[ "$DUMP_FILE" = /* ]] ; then
@@ -24,18 +33,19 @@ elif ! [[ -f "$DUMP_FILE" ]] ; then
 fi
 
 if [[ "$1" == "-k" || "$1" == "--kill-connections" ]] ; then
-  echo "TERMINATING DATABASE CONNECTIONS..."
-  sudo -u postgres psql -c "SELECT pg_terminate_backend(pid) 
+    echo "TERMINATING DATABASE CONNECTIONS..."
+    psql -U $DB_OWNER -c "SELECT pg_terminate_backend(pid) 
     FROM pg_stat_activity 
-    WHERE pid <> pg_backend_pid() AND datname = '$DB_NAME';"
+    WHERE pid <> pg_backend_pid() AND datname = '$DB_NAME';" $DB_NAME
 fi
 
-sudo -u postgres /bin/bash -c "echo DROPPING DATABASE... \
-                    && psql -c 'DROP DATABASE $DB_NAME;'; \
-                    \
-                    echo RECREATING DATABASE... \
-                    && psql -c 'CREATE DATABASE $DB_NAME OWNER $DB_OWNER;' \
-                    \
-                    && echo LOADING DATABASE... \
-                    && $BIN -d $DB_NAME $DUMP_FILE; \
-                    psql -c 'ALTER SCHEMA public OWNER TO $DB_OWNER;' $DB_NAME;"
+echo DROPPING DATABASE... \
+&& psql -U $DB_OWNER -c "DROP DATABASE $DB_NAME;" $DB_NAME
+
+echo RECREATING DATABASE... \
+&& psql -U $DB_OWNER -c "CREATE DATABASE $DB_NAME OWNER $DB_OWNER;" $DB_NAME
+
+echo LOADING DATABASE... \
+&& $BIN -d $DB_NAME $DUMP_FILE
+
+psql -U $DB_OWNER -c "ALTER SCHEMA public OWNER TO $DB_OWNER;" $DB_NAME

coldfront/api/statistics/tests/test_job_view_set.py

@@ -1154,7 +1154,7 @@ def test_post_allocation_missing_dates(self):
                 self.assertFalse(
                     Job.objects.filter(jobslurmid=data['jobslurmid']).exists())
 
-    @patch('coldfront.api.statistics.views.JobViewSet.validate_job_dates')
+    @patch('coldfront.api.statistics.views.validate_job_dates')
     def test_post_handles_exception_when_validating_dates(self, mock_method):
         """Test that a POST (create) request does not fail to create a
         job even if an exception is raised when determining whether
@@ -1396,7 +1396,7 @@ def test_put_allocation_missing_dates(self):
                 self.assertFalse(
                     Job.objects.filter(jobslurmid=data['jobslurmid']).exists())
 
-    @patch('coldfront.api.statistics.views.JobViewSet.validate_job_dates')
+    @patch('coldfront.api.statistics.views.validate_job_dates')
     def test_put_handles_exception_when_validating_dates(self, mock_method):
         """Test that a PUT (update) request does not fail to create a
         job even if an exception is raised when determining whether