diff --git a/database_management/tnris_org_local_time.sql b/database_management/tnris_org_local_time.sql new file mode 100644 index 00000000..8c1ed88d --- /dev/null +++ b/database_management/tnris_org_local_time.sql @@ -0,0 +1,14 @@ +-- this sql needs to be run to create the rest endpoint properly formatted date/time for training courses + +select title, start_date_time, end_date_time, +to_char(start_date_time at time zone 'America/Chicago', 'DD-MON-YYYY HH12:MI'), +to_char(end_date_time at time zone 'America/Chicago', 'DD-MON-YYYY HH12:MI' ) +from tnris_training + +-- returns two new columns. one column example: 'Monday October 21 08:00'. second column example ' - 12:00 PM'. +-- mash these two columsn together to get proper formatted date/time for tnris.org front end +-- final example format: 'Monday October 21 08:00 - 12:00 PM' +select title, start_date_time, end_date_time, +to_char(start_date_time at time zone 'America/Chicago', 'DayMonthDD HH24:MI'), +to_char(end_date_time at time zone 'America/Chicago', '- HH24:MI PM' ) +from tnris_forum_training diff --git a/etl/tnris_org.py b/etl/tnris_org.py new file mode 100644 index 00000000..d15919a5 --- /dev/null +++ b/etl/tnris_org.py @@ -0,0 +1,143 @@ +''' +1) get_s3_images function iterates through all data.tnris.org s3 bucket collection ids (keys), +copies each image and renames the copy to a new uuid. + +2) delete_old function to delete the old image name jpgs in s3. + +3) update_db function adds record to Django image table with: + - image_id = new uuid generated from get_s3_images function + - collection_id = collection_id + - image_url = new url + this function also replaces the existing collection table thumbnail_image records with the new uuid names. +''' +import os +import boto3, uuid +import psycopg2 +import datetime + +# s3 variables used in all three functions +client = boto3.client('s3') + +# iterate through, copy existing images and rename with uuid +def s3_images_to_db(bucket): + + kwargs = {'Bucket': bucket} + docs = 'documents/' + images = 'images/' + unique_images = [] + unique_docs = [] + total_count = 0 + img_count = 0 + doc_count = 0 + slash_count = 0 + + base_url = 'https://tnris-org-static.s3.amazonaws.com/' + + # Database Connection Info + database = os.environ.get('DB_NAME') + username = os.environ.get('DB_USER') + password = os.environ.get('DB_PASSWORD') + host = os.environ.get('DB_HOST') + port = os.environ.get('DB_PORT') + + # connection string + conn_string = "dbname='%s' user='%s' host='%s' password='%s' port='%s'" % (database, username, host, password, port) + + # db table names + image_table = 'tnris_image_url' + doc_table = 'tnris_doc_url' + + # connect to the database + conn = psycopg2.connect(conn_string) + cur = conn.cursor() + + # sql statement to get both collection_id and thumbnail_image fields + image_query = "SELECT image_id, image_name, image_url FROM %s;" % image_table + doc_query = "SELECT doc_id, doc_name, doc_url FROM %s;" % doc_table + + # execute sql statements + # cur.execute(image_query) + cur.execute(doc_query) + + # get response from both collection and image table queries + db_response = cur.fetchall() + + fixes = [] + + while True: + resp = client.list_objects_v2(**kwargs) + + for obj in resp['Contents']: + key = obj['Key'] + total_count += 1 + + # if key.startswith(images) and key not in unique_images and key is not None: + # unique_images.append(base_url + key.strip()) + # img_count += 1 + # image_id = uuid.uuid4() + # image_name = key.rsplit('/')[-1] + # print(image_name) + # image_url = base_url + key.strip() + # timestamp = datetime.datetime.now() + # + # # update the aws postgres rds db with new image names/urls + # cur.execute("INSERT INTO {table} ({id},{name},{url},{create},{modify}) VALUES ('{v1}','{v2}','{v3}','{v4}','{v5}');".format( + # table=image_table, + # id='image_id', + # name='image_name', + # url='image_url', + # create='created', + # modify='last_modified', + # v1=image_id, + # v2=image_name, + # v3=image_url, + # v4=timestamp, + # v5=timestamp) + # ) + # try: + # conn.commit() + # except: + # fixes.append(key) + + if key.startswith(docs) and key not in unique_docs and key is not None: + unique_docs.append(base_url + key.strip()) + doc_count += 1 + doc_id = uuid.uuid4() + doc_name = key.rsplit('/')[-1] + print(doc_name) + doc_url = base_url + key.strip() + timestamp = datetime.datetime.now() + + # update the aws postgres rds db with new image names/urls + cur.execute("INSERT INTO {table} ({id},{name},{url},{create},{modify}) VALUES ('{v1}','{v2}','{v3}','{v4}','{v5}');".format( + table=doc_table, + id='doc_id', + name='doc_name', + url='doc_url', + create='created', + modify='last_modified', + v1=doc_id, + v2=doc_name, + v3=doc_url, + v4=timestamp, + v5=timestamp) + ) + try: + conn.commit() + except: + fixes.append(key) + + try: + kwargs['ContinuationToken'] = resp['NextContinuationToken'] + except KeyError: + break + + print('total count ---->', total_count, 'images:', img_count, 'docs:', doc_count) + + print(fixes) + + # print('unique images:', unique_images) + # print('unique docs:', unique_docs) + +# execute function +s3_images_to_db('tnris-org-static') diff --git a/src/data_hub/dashboard.py b/src/data_hub/dashboard.py index 864ff4be..08d1fefb 100644 --- a/src/data_hub/dashboard.py +++ b/src/data_hub/dashboard.py @@ -43,11 +43,15 @@ def init_with_context(self, context): 'lore.models.FrameSize', 'lore.models.Scale', 'lore.models.County', - 'lore.models.Collection', + 'lore.models.Collection', 'msd.models.MapCollection', 'msd.models.MapDownload', 'msd.models.MapSize', - 'msd.models.PixelsPerInch'), + 'msd.models.PixelsPerInch', + 'tnris_org.models.TnrisImage', + 'tnris_org.models.TnrisDocument', + 'tnris_org.models.TnrisTraining', + 'tnris_org.models.TnrisForumTraining'), )) self.children.append(modules.AppList( @@ -89,6 +93,17 @@ def init_with_context(self, context): 'msd.models.PixelsPerInch'), )) + self.children.append(modules.AppList( + title='Website Content', + collapsible=True, + column=1, + css_classes=('grp-collapse grp-closed',), + models=('tnris_org.models.TnrisImage', + 'tnris_org.models.TnrisDocument', + 'tnris_org.models.TnrisTraining', + 'tnris_org.models.TnrisForumTraining'), + )) + # append a recent actions module self.children.append(modules.RecentActions( title='Recent Actions', diff --git a/src/data_hub/data_hub/settings.py b/src/data_hub/data_hub/settings.py index 17ecd6b0..0dfb3c07 100644 --- a/src/data_hub/data_hub/settings.py +++ b/src/data_hub/data_hub/settings.py @@ -24,7 +24,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY', 'emla71m=n&u^4_=@07&i8@oyw1thl%bc7x9dqjx7=l1r^d77+=') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = [ 'data.tnris.org', @@ -65,6 +65,7 @@ 'lcd', 'lore', 'msd', + 'tnris_org', 'corsheaders', 'django_admin_listfilter_dropdown', 'django.contrib.contenttypes', diff --git a/src/data_hub/lcd/urls.py b/src/data_hub/lcd/urls.py index bc423336..6ee6fe6b 100644 --- a/src/data_hub/lcd/urls.py +++ b/src/data_hub/lcd/urls.py @@ -21,6 +21,7 @@ from .views import resource_update_progress import lore import msd +import tnris_org router = routers.DefaultRouter(trailing_slash=False) @@ -36,5 +37,6 @@ path('data_hub-auth/?', include('rest_framework.urls', namespace='lcd_rest_framework')), path('resource-update-progress/', resource_update_progress, name='resource-update-progress'), path('historical/', include('lore.urls')), - path('map/', include('msd.urls')) + path('map/', include('msd.urls')), + path('tnris_org/', include('tnris_org.urls')) ] diff --git a/src/data_hub/tnris_org/__init__.py b/src/data_hub/tnris_org/__init__.py new file mode 100644 index 00000000..37c6cd7d --- /dev/null +++ b/src/data_hub/tnris_org/__init__.py @@ -0,0 +1 @@ +default_app_config = 'tnris_org.apps.TnrisOrgConfig' diff --git a/src/data_hub/tnris_org/admin.py b/src/data_hub/tnris_org/admin.py new file mode 100644 index 00000000..0ef5707d --- /dev/null +++ b/src/data_hub/tnris_org/admin.py @@ -0,0 +1,51 @@ +from django.contrib import admin + +# Register your models here. +from .forms import ( + ImageForm, + DocumentForm, + TnrisTrainingForm, + TnrisForumTrainingForm +) +from .models import ( + TnrisImage, + TnrisDocument, + TnrisTraining, + TnrisForumTraining +) + + +@admin.register(TnrisImage) +class TnrisImageAdmin(admin.ModelAdmin): + model = TnrisImage + form = ImageForm + ordering = ('image_name',) + list_display = ('image_name', 'image_url', 'created') + search_fields = ('image_name', 'image_url') + + +@admin.register(TnrisDocument) +class TnrisDocumentAdmin(admin.ModelAdmin): + model = TnrisDocument + form = DocumentForm + ordering = ('document_name',) + list_display = ('document_name', 'document_url', 'created') + search_fields = ('document_name', 'document_url') + + +@admin.register(TnrisTraining) +class TnrisTrainingAdmin(admin.ModelAdmin): + model = TnrisTraining + form = TnrisTrainingForm + ordering = ('title',) + list_display = ('title', 'instructor', 'start_date_time', 'end_date_time') + search_fields = ('title', 'instructor') + + +@admin.register(TnrisForumTraining) +class TnrisForumTrainingAdmin(admin.ModelAdmin): + model = TnrisForumTraining + form = TnrisForumTrainingForm + ordering = ('title',) + list_display = ('title', 'instructor', 'start_date_time', 'end_date_time') + search_fields = ('title', 'instructor') diff --git a/src/data_hub/tnris_org/apps.py b/src/data_hub/tnris_org/apps.py new file mode 100644 index 00000000..6c888161 --- /dev/null +++ b/src/data_hub/tnris_org/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class TnrisOrgConfig(AppConfig): + name = 'tnris_org' + verbose_name = 'TNRIS.org' diff --git a/src/data_hub/tnris_org/forms.py b/src/data_hub/tnris_org/forms.py new file mode 100644 index 00000000..d4cd69d6 --- /dev/null +++ b/src/data_hub/tnris_org/forms.py @@ -0,0 +1,150 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.contrib.admin.widgets import AdminDateWidget + +from string import Template +from django.utils.safestring import mark_safe + +from django.db.utils import ProgrammingError +from .models import (TnrisImage, TnrisDocument, TnrisTraining, TnrisForumTraining) +import os +import boto3, uuid + + +class PictureWidget(forms.widgets.Widget): + def render(self, name, value, attrs=None): + if value is None: + html = Template("""""") + else: + html = Template("""""") + return mark_safe(html.substitute(link=value,name=name)) + + +class DocumentWidget(forms.widgets.Widget): + def render(self, name, value, attrs=None): + if value is None: + html = Template("""""") + else: + html = Template("""""") + return mark_safe(html.substitute(link=value,name=name)) + + +class ImageForm(forms.ModelForm): + class Meta: + model = TnrisImage + fields = ('__all__') + + image_url = forms.FileField(required=False, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record.") + + # boto3 s3 object + client = boto3.client('s3') + + # function to upload image to s3 and update dbase link + def handle_image(self, field, file): + # upload image + key = "images/%s" % (file) + response = self.client.put_object( + Bucket='tnris-org-static', + ACL='public-read', + ContentType='image', + Key=key, + Body=file + ) + print('%s upload success!' % key) + # update link in database table + setattr(self.instance, field, "https://tnris-org-static.s3.amazonaws.com/" + key) + setattr(self.instance, "image_name", str(file)) + return + + # custom handling of images on save + def clean(self, commit=True): + # check for files + files = self.files + for f in files: + print(str(files[f])) + ext = os.path.splitext(str(files[f]))[1] + valid_extensions = ['.jpg', '.png', '.gif', '.jpeg', '.svg'] + # validation to prevent non-standard image formats or other files from being uploaded + if not ext.lower() in valid_extensions: + raise ValidationError(u"Unsupported file extension. Only .jpg, .png, .gif, and .jpeg file extensions supported for Tnris Images") + # validation to check if image file name already exists in database + name_set = TnrisImage.objects.filter(image_name=str(files[f])) + if len(name_set) > 0: + raise ValidationError(u"Image file name already exists. Rename your file.") + + self.handle_image(f, files[f]) + + return super(ImageForm, self).save(commit=commit) + + +class DocumentForm(forms.ModelForm): + class Meta: + model = TnrisDocument + fields = ('__all__') + + document_url = forms.FileField(required=False, widget=DocumentWidget, help_text="Choose a document file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record.") + + # boto3 s3 object + client = boto3.client('s3') + + # function to upload document to s3 and update dbase link + def handle_doc(self, field, file): + # upload image + key = "documents/%s" % (file) + response = self.client.put_object( + Bucket='tnris-org-static', + ACL='public-read', + Key=key, + Body=file + ) + print('%s upload success!' % key) + # update link in database table + setattr(self.instance, field, "https://tnris-org-static.s3.amazonaws.com/" + key) + setattr(self.instance, "document_name", str(file)) + return + + # custom handling of documents on save + def clean(self, commit=True): + # check for files + files = self.files + for f in files: + print(str(files[f])) + ext = os.path.splitext(str(files[f]))[1] + invalid_extensions = ['.jpg', '.png', '.gif', '.jpeg', '.svg'] + # validation to prevent image formats from being uploaded + if ext.lower() in invalid_extensions: + raise ValidationError(u"Unsupported file extension. All images should be uploaded to 'Tnris Images', only document type files should be uploaded here.") + # validation to check if document file name already exists in database + name_set = TnrisDocument.objects.filter(document_name=str(files[f])) + if len(name_set) > 0: + raise ValidationError(u"Document file name already exists. Rename your file.") + + self.handle_doc(f, files[f]) + + return super(DocumentForm, self).save(commit=commit) + + +class TnrisTrainingForm(forms.ModelForm): + class Meta: + model = TnrisTraining + fields = ('__all__') + + start_date_time = forms.DateTimeField(help_text="Accepted date and time input formats: '10/25/06 14:30', '10/25/2006 14:30', '2006-10-25 14:30'") + end_date_time = forms.DateTimeField(help_text="Accepted date and time input formats: '10/25/06 14:30', '10/25/2006 14:30', '2006-10-25 14:30'") + cost = forms.DecimalField(help_text="Example of accepted formats for training cost: '50.00', '999', '99.99'. Max of 6 digits and 2 decimal places.") + registration_open = forms.BooleanField(required=False, help_text="Check the box to change registration to open. Default is unchecked.") + public = forms.BooleanField(required=False, help_text="Check the box to make this training record visible on the website. Default is unchecked.") + max_students = forms.IntegerField(required=False, help_text="Enter max number of students for class room.") + + +class TnrisForumTrainingForm(forms.ModelForm): + class Meta: + model = TnrisForumTraining + fields = ('__all__') + + start_date_time = forms.DateTimeField(help_text="Accepted date and time input formats: '10/25/06 14:30', '10/25/2006 14:30', '2006-10-25 14:30'") + end_date_time = forms.DateTimeField(help_text="Accepted date and time input formats: '10/25/06 14:30', '10/25/2006 14:30', '2006-10-25 14:30'") + cost = forms.DecimalField(help_text="Example of accepted formats for training cost: '50.00', '999', '99.99'. Max of 6 digits and 2 decimal places.") + registration_open = forms.BooleanField(required=False, help_text="Check the box to change registration to open. Default is unchecked.") + public = forms.BooleanField(required=False, help_text="Check the box to make this training record visible on the website. Default is unchecked.") + max_students = forms.IntegerField(required=False, help_text="Enter max number of students for class room.") diff --git a/src/data_hub/tnris_org/migrations/0001_initial.py b/src/data_hub/tnris_org/migrations/0001_initial.py new file mode 100644 index 00000000..239c35dd --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 2.0.13 on 2019-07-10 14:46 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TnrisDocUrl', + fields=[ + ('doc_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Document ID')), + ('doc_name', models.CharField(max_length=200, verbose_name='Document Name')), + ('doc_url', models.URLField(max_length=255, verbose_name='Document URL')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')), + ], + options={ + 'verbose_name': 'Tnris Document Url', + 'verbose_name_plural': 'Tnris Document Urls', + 'db_table': 'tnris_doc_url', + }, + ), + migrations.CreateModel( + name='TnrisImageUrl', + fields=[ + ('image_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Image ID')), + ('image_name', models.CharField(max_length=200, verbose_name='Image Name')), + ('image_url', models.URLField(max_length=255, verbose_name='Image URL')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')), + ], + options={ + 'verbose_name': 'Tnris Image Url', + 'verbose_name_plural': 'Tnris Image Urls', + 'db_table': 'tnris_image_url', + }, + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0002_auto_20190710_1153.py b/src/data_hub/tnris_org/migrations/0002_auto_20190710_1153.py new file mode 100644 index 00000000..8fff7406 --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0002_auto_20190710_1153.py @@ -0,0 +1,23 @@ +# Generated by Django 2.0.13 on 2019-07-10 16:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='tnrisdocurl', + name='doc_name', + field=models.CharField(editable=False, max_length=200, verbose_name='Document Name'), + ), + migrations.AlterField( + model_name='tnrisimageurl', + name='image_name', + field=models.CharField(editable=False, max_length=200, verbose_name='Image Name'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0003_auto_20190711_1119.py b/src/data_hub/tnris_org/migrations/0003_auto_20190711_1119.py new file mode 100644 index 00000000..fcea919f --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0003_auto_20190711_1119.py @@ -0,0 +1,52 @@ +# Generated by Django 2.0.13 on 2019-07-11 16:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0002_auto_20190710_1153'), + ] + + operations = [ + migrations.RenameModel( + old_name='TnrisDocUrl', + new_name='TnrisDocument', + ), + migrations.RenameModel( + old_name='TnrisImageUrl', + new_name='TnrisImage', + ), + migrations.AlterModelOptions( + name='tnrisdocument', + options={'verbose_name': 'Tnris Document', 'verbose_name_plural': 'Tnris Documents'}, + ), + migrations.AlterModelOptions( + name='tnrisimage', + options={'verbose_name': 'Tnris Image', 'verbose_name_plural': 'Tnris Images'}, + ), + migrations.RenameField( + model_name='tnrisdocument', + old_name='doc_id', + new_name='document_id', + ), + migrations.RenameField( + model_name='tnrisdocument', + old_name='doc_name', + new_name='document_name', + ), + migrations.RenameField( + model_name='tnrisdocument', + old_name='doc_url', + new_name='document_url', + ), + migrations.AlterModelTable( + name='tnrisdocument', + table='tnris_document', + ), + migrations.AlterModelTable( + name='tnrisimage', + table='tnris_image', + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0004_tnrisforumtraining_tnristraining.py b/src/data_hub/tnris_org/migrations/0004_tnrisforumtraining_tnristraining.py new file mode 100644 index 00000000..beee729a --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0004_tnrisforumtraining_tnristraining.py @@ -0,0 +1,61 @@ +# Generated by Django 2.0.13 on 2019-07-16 19:39 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0003_auto_20190711_1119'), + ] + + operations = [ + migrations.CreateModel( + name='TnrisForumTraining', + fields=[ + ('training_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Training ID')), + ('training_day', models.PositiveSmallIntegerField(verbose_name='Forum Training Day')), + ('title', models.CharField(max_length=255, verbose_name='Training Title')), + ('start_date_time', models.DateTimeField(verbose_name='Training Start Date & Time')), + ('end_date_time', models.DateTimeField(verbose_name='Training End Date & Time')), + ('instructor', models.CharField(max_length=100, verbose_name='Training Instructor')), + ('cost', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='Training Cost')), + ('registration_open', models.BooleanField(default=False, verbose_name='Registration Open')), + ('location', models.CharField(max_length=255, verbose_name='Training Location')), + ('room', models.CharField(max_length=255, verbose_name='Training Room')), + ('max_students', models.PositiveSmallIntegerField(verbose_name='Max Student Amount')), + ('instructor_bio', models.TextField(verbose_name='Training Instructor Bio')), + ('description', models.TextField(verbose_name='Training Description')), + ('teaser', models.TextField(verbose_name='Training Teaser')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')), + ], + options={ + 'verbose_name': 'Tnris Forum Training', + 'verbose_name_plural': 'Tnris Forum Trainings', + 'db_table': 'tnris_forum_training', + }, + ), + migrations.CreateModel( + name='TnrisTraining', + fields=[ + ('training_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Training ID')), + ('start_date_time', models.DateTimeField(verbose_name='Training Start Date & Time')), + ('end_date_time', models.DateTimeField(verbose_name='Training End Date & Time')), + ('title', models.CharField(max_length=255, verbose_name='Training Title')), + ('instructor', models.CharField(max_length=100, verbose_name='Training Instructor')), + ('cost', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='Training Cost')), + ('registration_open', models.BooleanField(default=False, verbose_name='Registration Open')), + ('instructor_bio', models.TextField(verbose_name='Training Instructor Bio')), + ('description', models.TextField(verbose_name='Training Description')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')), + ], + options={ + 'verbose_name': 'Tnris Training', + 'verbose_name_plural': 'Tnris Trainings', + 'db_table': 'tnris_training', + }, + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0005_auto_20190716_1531.py b/src/data_hub/tnris_org/migrations/0005_auto_20190716_1531.py new file mode 100644 index 00000000..f3a79efd --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0005_auto_20190716_1531.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2019-07-16 20:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0004_tnrisforumtraining_tnristraining'), + ] + + operations = [ + migrations.AlterField( + model_name='tnrisforumtraining', + name='instructor_bio', + field=models.TextField(blank=True, verbose_name='Training Instructor Bio'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0006_auto_20190716_1532.py b/src/data_hub/tnris_org/migrations/0006_auto_20190716_1532.py new file mode 100644 index 00000000..7ac78527 --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0006_auto_20190716_1532.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2019-07-16 20:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0005_auto_20190716_1531'), + ] + + operations = [ + migrations.AlterField( + model_name='tnristraining', + name='instructor_bio', + field=models.TextField(blank=True, verbose_name='Training Instructor Bio'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0007_auto_20190716_1543.py b/src/data_hub/tnris_org/migrations/0007_auto_20190716_1543.py new file mode 100644 index 00000000..05da9665 --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0007_auto_20190716_1543.py @@ -0,0 +1,33 @@ +# Generated by Django 2.0.13 on 2019-07-16 20:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0006_auto_20190716_1532'), + ] + + operations = [ + migrations.AddField( + model_name='tnrisforumtraining', + name='public', + field=models.BooleanField(default=False, verbose_name='Public'), + ), + migrations.AddField( + model_name='tnristraining', + name='public', + field=models.BooleanField(default=False, verbose_name='Public'), + ), + migrations.AlterField( + model_name='tnrisforumtraining', + name='max_students', + field=models.PositiveSmallIntegerField(blank=True, verbose_name='Max Student Amount'), + ), + migrations.AlterField( + model_name='tnrisforumtraining', + name='teaser', + field=models.TextField(blank=True, verbose_name='Training Teaser'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0008_auto_20190717_0733.py b/src/data_hub/tnris_org/migrations/0008_auto_20190717_0733.py new file mode 100644 index 00000000..4517be05 --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0008_auto_20190717_0733.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2019-07-17 12:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0007_auto_20190716_1543'), + ] + + operations = [ + migrations.AlterField( + model_name='tnrisforumtraining', + name='room', + field=models.CharField(blank=True, max_length=255, verbose_name='Training Room'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0009_auto_20190717_1453.py b/src/data_hub/tnris_org/migrations/0009_auto_20190717_1453.py new file mode 100644 index 00000000..a8fd0b8f --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0009_auto_20190717_1453.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2019-07-17 19:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0008_auto_20190717_0733'), + ] + + operations = [ + migrations.AlterField( + model_name='tnrisforumtraining', + name='max_students', + field=models.PositiveSmallIntegerField(blank=True, null=True, verbose_name='Max Student Amount'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/__init__.py b/src/data_hub/tnris_org/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/data_hub/tnris_org/models.py b/src/data_hub/tnris_org/models.py new file mode 100644 index 00000000..00fa8ba8 --- /dev/null +++ b/src/data_hub/tnris_org/models.py @@ -0,0 +1,265 @@ +from django.db import models + +import uuid +import boto3 +import os + + +""" +********** TNRIS.ORG Static File/Resource Tables (Images, Docs, Training) ********** +""" + + +class TnrisImage(models.Model): + """Tnris.org image resource table containing an S3 bucket url for each image""" + + class Meta: + db_table = 'tnris_image' + verbose_name = 'Tnris Image' + verbose_name_plural = 'Tnris Images' + + image_id = models.UUIDField( + 'Image ID', + primary_key=True, + default=uuid.uuid4, + editable=False + ) + image_name = models.CharField( + 'Image Name', + max_length=200, + editable=False + ) + image_url = models.URLField( + 'Image URL', + max_length=255 + ) + created = models.DateTimeField( + 'Created', + auto_now_add=True + ) + last_modified = models.DateTimeField( + 'Last Modified', + auto_now=True + ) + # delete s3 image files + def delete(self, *args, **kwargs): + client = boto3.client('s3') + key = str(self).replace('https://tnris-org-static.s3.amazonaws.com/', '') + print(key, 'successfully deleted') + response = client.delete_object( + Bucket='tnris-org-static', + Key=key + ) + print(self) + super().delete(*args, **kwargs) + + def __str__(self): + return self.image_url + + +class TnrisDocument(models.Model): + """Tnris.org document resource table containing an S3 bucket url for each document""" + + class Meta: + db_table = 'tnris_document' + verbose_name = 'Tnris Document' + verbose_name_plural = 'Tnris Documents' + + document_id = models.UUIDField( + 'Document ID', + primary_key=True, + default=uuid.uuid4, + editable=False + ) + document_name = models.CharField( + 'Document Name', + max_length=200, + editable=False + ) + document_url = models.URLField( + 'Document URL', + max_length=255, + ) + created = models.DateTimeField( + 'Created', + auto_now_add=True + ) + last_modified = models.DateTimeField( + 'Last Modified', + auto_now=True + ) + # delete s3 doc files + def delete(self, *args, **kwargs): + client = boto3.client('s3') + key = str(self).replace('https://tnris-org-static.s3.amazonaws.com/', '') + print(key, 'successfully deleted') + response = client.delete_object( + Bucket='tnris-org-static', + Key=key + ) + print(self) + super().delete(*args, **kwargs) + + def __str__(self): + return self.document_url + + +class TnrisTraining(models.Model): + """Regular training courses scheduled by Tnris""" + + class Meta: + db_table = 'tnris_training' + verbose_name = 'Tnris Training' + verbose_name_plural = 'Tnris Trainings' + + training_id = models.UUIDField( + 'Training ID', + primary_key=True, + default=uuid.uuid4, + editable=False, + blank=False + ) + start_date_time = models.DateTimeField( + 'Training Start Date & Time', + blank=False + ) + end_date_time = models.DateTimeField( + 'Training End Date & Time', + blank=False + ) + title = models.CharField( + 'Training Title', + max_length=255, + blank=False + ) + instructor = models.CharField( + 'Training Instructor', + max_length=100, + blank=False + ) + cost = models.DecimalField( + 'Training Cost', + max_digits=6, + decimal_places=2, + blank=False + ) + registration_open = models.BooleanField( + 'Registration Open', + default=False, + null=False + ) + public = models.BooleanField( + 'Public', + default=False, + null=False + ) + instructor_bio = models.TextField( + 'Training Instructor Bio', + blank=True + ) + description = models.TextField( + 'Training Description' + ) + created = models.DateTimeField( + 'Created', + auto_now_add=True + ) + last_modified = models.DateTimeField( + 'Last Modified', + auto_now=True + ) + + def __str__(self): + return self.title + + +class TnrisForumTraining(models.Model): + """Texas GIS Forum training courses scheduled by Tnris""" + + class Meta: + db_table = 'tnris_forum_training' + verbose_name = 'Tnris Forum Training' + verbose_name_plural = 'Tnris Forum Trainings' + + training_id = models.UUIDField( + 'Training ID', + primary_key=True, + default=uuid.uuid4, + editable=False, + blank=False + ) + training_day = models.PositiveSmallIntegerField( + 'Forum Training Day', + blank=False + ) + title = models.CharField( + 'Training Title', + max_length=255, + blank=False + ) + start_date_time = models.DateTimeField( + 'Training Start Date & Time', + blank=False + ) + end_date_time = models.DateTimeField( + 'Training End Date & Time', + blank=False + ) + instructor = models.CharField( + 'Training Instructor', + max_length=100, + blank=False + ) + cost = models.DecimalField( + 'Training Cost', + max_digits=6, + decimal_places=2, + blank=False + ) + registration_open = models.BooleanField( + 'Registration Open', + default=False, + null=False + ) + public = models.BooleanField( + 'Public', + default=False, + null=False + ) + location = models.CharField( + 'Training Location', + max_length=255, + blank=False + ) + room = models.CharField( + 'Training Room', + max_length=255, + blank=True + ) + max_students = models.PositiveSmallIntegerField( + 'Max Student Amount', + blank=True, + null=True + ) + instructor_bio = models.TextField( + 'Training Instructor Bio', + blank=True + ) + description = models.TextField( + 'Training Description' + ) + teaser = models.TextField( + 'Training Teaser', + blank=True + ) + created = models.DateTimeField( + 'Created', + auto_now_add=True + ) + last_modified = models.DateTimeField( + 'Last Modified', + auto_now=True + ) + + def __str__(self): + return self.title diff --git a/src/data_hub/tnris_org/serializers.py b/src/data_hub/tnris_org/serializers.py new file mode 100644 index 00000000..0ced55ef --- /dev/null +++ b/src/data_hub/tnris_org/serializers.py @@ -0,0 +1,24 @@ +from rest_framework import serializers + +from .models import (TnrisTraining, TnrisForumTraining) +from datetime import datetime + + +class TnrisTrainingSerializer(serializers.ModelSerializer): + class Meta: + model = TnrisTraining + fields = '__all__' + + # format date/time for api rest endpoint to use on front end + start_date_time = serializers.DateTimeField(format="%A, %B %d %I:%M %p") + end_date_time = serializers.DateTimeField(format="%A, %B %d %I:%M %p") + + +class TnrisForumTrainingSerializer(serializers.ModelSerializer): + class Meta: + model = TnrisForumTraining + fields = '__all__' + + # format date/time for api rest endpoint to use on front end + start_date_time = serializers.DateTimeField(format="%A, %B %d %I:%M %p") + end_date_time = serializers.DateTimeField(format="%A, %B %d %I:%M %p") diff --git a/src/data_hub/tnris_org/tests.py b/src/data_hub/tnris_org/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/data_hub/tnris_org/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/data_hub/tnris_org/urls.py b/src/data_hub/tnris_org/urls.py new file mode 100644 index 00000000..8f8e97b3 --- /dev/null +++ b/src/data_hub/tnris_org/urls.py @@ -0,0 +1,32 @@ +"""master systems display (msd) URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.11/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.urls import path, include +from rest_framework import routers +from rest_framework_swagger.views import get_swagger_view +from rest_framework.schemas import get_schema_view + +from .viewsets import (TnrisTrainingViewSet, TnrisForumTrainingViewSet) + +router = routers.DefaultRouter(trailing_slash=False) +router.register(r'training/?', TnrisTrainingViewSet, base_name="TnrisTraining") +router.register(r'forum_training/?', TnrisForumTrainingViewSet, base_name="TnrisForumTraining") + +schema_view = get_swagger_view(title='Maps API') + +urlpatterns = [ + path('', include(router.urls)), + path('schema/', schema_view) +] diff --git a/src/data_hub/tnris_org/views.py b/src/data_hub/tnris_org/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/src/data_hub/tnris_org/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/src/data_hub/tnris_org/viewsets.py b/src/data_hub/tnris_org/viewsets.py new file mode 100644 index 00000000..0bf48270 --- /dev/null +++ b/src/data_hub/tnris_org/viewsets.py @@ -0,0 +1,44 @@ +from rest_framework import viewsets +from rest_framework.response import Response + +from .models import TnrisTraining, TnrisForumTraining +from .serializers import (TnrisTrainingSerializer, TnrisForumTrainingSerializer) + +class TnrisTrainingViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = TnrisTrainingSerializer + http_method_names = ['get'] + + def get_queryset(self): + args = {'public': True} + null_list = ['null', 'Null', 'none', 'None'] + # create argument object of query clauses + for field in self.request.query_params.keys(): + if field != 'limit' and field != 'offset': + value = self.request.query_params.get(field) + # convert null queries + if value in null_list: + value = None + args[field] = value + # get records using query + queryset = TnrisTraining.objects.filter(**args).order_by('title', 'start_date_time', 'end_date_time') + return queryset + + +class TnrisForumTrainingViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = TnrisForumTrainingSerializer + http_method_names = ['get'] + + def get_queryset(self): + args = {'public': True} + null_list = ['null', 'Null', 'none', 'None'] + # create argument object of query clauses + for field in self.request.query_params.keys(): + if field != 'limit' and field != 'offset': + value = self.request.query_params.get(field) + # convert null queries + if value in null_list: + value = None + args[field] = value + # get records using query + queryset = TnrisForumTraining.objects.filter(**args).order_by('title', 'start_date_time', 'end_date_time') + return queryset