Merge branch 'develop'

database_management/tnris_org_local_time.sql

@@ -0,0 +1,14 @@
+-- this sql needs to be run to create the rest endpoint properly formatted date/time for training courses
+
+select title, start_date_time, end_date_time,
+to_char(start_date_time at time zone 'America/Chicago', 'DD-MON-YYYY HH12:MI'),
+to_char(end_date_time at time zone 'America/Chicago', 'DD-MON-YYYY HH12:MI' )
+from tnris_training
+
+-- returns two new columns. one column example: 'Monday October 21 08:00'. second column example ' - 12:00 PM'.
+-- mash these two columsn together to get proper formatted date/time for tnris.org front end
+-- final example format: 'Monday October 21 08:00 - 12:00 PM'
+select title, start_date_time, end_date_time,
+to_char(start_date_time at time zone 'America/Chicago', 'DayMonthDD HH24:MI'),
+to_char(end_date_time at time zone 'America/Chicago', '- HH24:MI PM' )
+from tnris_forum_training

etl/tnris_org.py

@@ -0,0 +1,143 @@
+'''
+1) get_s3_images function iterates through all data.tnris.org s3 bucket collection ids (keys),
+copies each image and renames the copy to a new uuid.
+
+2) delete_old function to delete the old image name jpgs in s3.
+
+3) update_db function adds record to Django image table with:
+    - image_id = new uuid generated from get_s3_images function
+    - collection_id = collection_id
+    - image_url = new url
+   this function also replaces the existing collection table thumbnail_image records with the new uuid names.
+'''
+import os
+import boto3, uuid
+import psycopg2
+import datetime
+
+# s3 variables used in all three functions
+client = boto3.client('s3')
+
+# iterate through, copy existing images and rename with uuid
+def s3_images_to_db(bucket):
+
+    kwargs = {'Bucket': bucket}
+    docs = 'documents/'
+    images = 'images/'
+    unique_images = []
+    unique_docs = []
+    total_count = 0
+    img_count = 0
+    doc_count = 0
+    slash_count = 0
+
+    base_url = 'https://tnris-org-static.s3.amazonaws.com/'
+
+    # Database Connection Info
+    database = os.environ.get('DB_NAME')
+    username = os.environ.get('DB_USER')
+    password = os.environ.get('DB_PASSWORD')
+    host = os.environ.get('DB_HOST')
+    port = os.environ.get('DB_PORT')
+
+    # connection string
+    conn_string = "dbname='%s' user='%s' host='%s' password='%s' port='%s'" % (database, username, host, password, port)
+
+    # db table names
+    image_table = 'tnris_image_url'
+    doc_table = 'tnris_doc_url'
+
+    # connect to the database
+    conn = psycopg2.connect(conn_string)
+    cur = conn.cursor()
+
+    # sql statement to get both collection_id and thumbnail_image fields
+    image_query = "SELECT image_id, image_name, image_url FROM %s;" % image_table
+    doc_query = "SELECT doc_id, doc_name, doc_url FROM %s;" % doc_table
+
+    # execute sql statements
+    # cur.execute(image_query)
+    cur.execute(doc_query)
+
+    # get response from both collection and image table queries
+    db_response = cur.fetchall()
+
+    fixes = []
+
+    while True:
+        resp = client.list_objects_v2(**kwargs)
+
+        for obj in resp['Contents']:
+            key = obj['Key']
+            total_count += 1
+
+            # if key.startswith(images) and key not in unique_images and key is not None:
+            #     unique_images.append(base_url + key.strip())
+            #     img_count += 1
+            #     image_id = uuid.uuid4()
+            #     image_name = key.rsplit('/')[-1]
+            #     print(image_name)
+            #     image_url = base_url + key.strip()
+            #     timestamp = datetime.datetime.now()
+            #
+            #     # update the aws postgres rds db with new image names/urls
+            #     cur.execute("INSERT INTO {table} ({id},{name},{url},{create},{modify}) VALUES ('{v1}','{v2}','{v3}','{v4}','{v5}');".format(
+            #         table=image_table,
+            #         id='image_id',
+            #         name='image_name',
+            #         url='image_url',
+            #         create='created',
+            #         modify='last_modified',
+            #         v1=image_id,
+            #         v2=image_name,
+            #         v3=image_url,
+            #         v4=timestamp,
+            #         v5=timestamp)
+            #     )
+            #     try:
+            #         conn.commit()
+            #     except:
+            #         fixes.append(key)
+
+            if key.startswith(docs) and key not in unique_docs and key is not None:
+                unique_docs.append(base_url + key.strip())
+                doc_count += 1
+                doc_id = uuid.uuid4()
+                doc_name = key.rsplit('/')[-1]
+                print(doc_name)
+                doc_url = base_url + key.strip()
+                timestamp = datetime.datetime.now()
+
+                # update the aws postgres rds db with new image names/urls
+                cur.execute("INSERT INTO {table} ({id},{name},{url},{create},{modify}) VALUES ('{v1}','{v2}','{v3}','{v4}','{v5}');".format(
+                    table=doc_table,
+                    id='doc_id',
+                    name='doc_name',
+                    url='doc_url',
+                    create='created',
+                    modify='last_modified',
+                    v1=doc_id,
+                    v2=doc_name,
+                    v3=doc_url,
+                    v4=timestamp,
+                    v5=timestamp)
+                )
+                try:
+                    conn.commit()
+                except:
+                    fixes.append(key)
+
+        try:
+            kwargs['ContinuationToken'] = resp['NextContinuationToken']
+        except KeyError:
+            break
+
+    print('total count ---->', total_count, 'images:', img_count, 'docs:', doc_count)
+
+    print(fixes)
+
+    # print('unique images:', unique_images)
+    # print('unique docs:', unique_docs)
+
+# execute function
+s3_images_to_db('tnris-org-static')

src/data_hub/dashboard.py

@@ -43,11 +43,15 @@ def init_with_context(self, context):
                      'lore.models.FrameSize',
                      'lore.models.Scale',
                      'lore.models.County',
-                      'lore.models.Collection',
+                     'lore.models.Collection',
                      'msd.models.MapCollection',
                      'msd.models.MapDownload',
                      'msd.models.MapSize',
-                     'msd.models.PixelsPerInch'),
+                     'msd.models.PixelsPerInch',
+                     'tnris_org.models.TnrisImage',
+                     'tnris_org.models.TnrisDocument',
+                     'tnris_org.models.TnrisTraining',
+                     'tnris_org.models.TnrisForumTraining'),
         ))
 
         self.children.append(modules.AppList(
@@ -89,6 +93,17 @@ def init_with_context(self, context):
                     'msd.models.PixelsPerInch'),
         ))
 
+        self.children.append(modules.AppList(
+            title='Website Content',
+            collapsible=True,
+            column=1,
+            css_classes=('grp-collapse grp-closed',),
+            models=('tnris_org.models.TnrisImage',
+                    'tnris_org.models.TnrisDocument',
+                    'tnris_org.models.TnrisTraining',
+                    'tnris_org.models.TnrisForumTraining'),
+        ))
+
         # append a recent actions module
         self.children.append(modules.RecentActions(
             title='Recent Actions',

src/data_hub/data_hub/settings.py

@@ -24,7 +24,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY', 'emla71m=n&u^4_=@07&i8@oyw1thl%bc7x9dqjx7=l1r^d77+=')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = [
     'data.tnris.org',
@@ -65,6 +65,7 @@
     'lcd',
     'lore',
     'msd',
+    'tnris_org',
     'corsheaders',
     'django_admin_listfilter_dropdown',
     'django.contrib.contenttypes',

src/data_hub/lcd/urls.py

@@ -21,6 +21,7 @@
 from .views import resource_update_progress
 import lore
 import msd
+import tnris_org
 
 
 router = routers.DefaultRouter(trailing_slash=False)
@@ -36,5 +37,6 @@
     path('data_hub-auth/?', include('rest_framework.urls', namespace='lcd_rest_framework')),
     path('resource-update-progress/', resource_update_progress, name='resource-update-progress'),
     path('historical/', include('lore.urls')),
-    path('map/', include('msd.urls'))
+    path('map/', include('msd.urls')),
+    path('tnris_org/', include('tnris_org.urls'))
 ]

src/data_hub/tnris_org/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'tnris_org.apps.TnrisOrgConfig'

src/data_hub/tnris_org/admin.py

@@ -0,0 +1,51 @@
+from django.contrib import admin
+
+# Register your models here.
+from .forms import (
+    ImageForm,
+    DocumentForm,
+    TnrisTrainingForm,
+    TnrisForumTrainingForm
+)
+from .models import (
+    TnrisImage,
+    TnrisDocument,
+    TnrisTraining,
+    TnrisForumTraining
+)
+
+
+@admin.register(TnrisImage)
+class TnrisImageAdmin(admin.ModelAdmin):
+    model = TnrisImage
+    form = ImageForm
+    ordering = ('image_name',)
+    list_display = ('image_name', 'image_url', 'created')
+    search_fields = ('image_name', 'image_url')
+
+
+@admin.register(TnrisDocument)
+class TnrisDocumentAdmin(admin.ModelAdmin):
+    model = TnrisDocument
+    form = DocumentForm
+    ordering = ('document_name',)
+    list_display = ('document_name', 'document_url', 'created')
+    search_fields = ('document_name', 'document_url')
+
+
+@admin.register(TnrisTraining)
+class TnrisTrainingAdmin(admin.ModelAdmin):
+    model = TnrisTraining
+    form = TnrisTrainingForm
+    ordering = ('title',)
+    list_display = ('title', 'instructor', 'start_date_time', 'end_date_time')
+    search_fields = ('title', 'instructor')
+
+
+@admin.register(TnrisForumTraining)
+class TnrisForumTrainingAdmin(admin.ModelAdmin):
+    model = TnrisForumTraining
+    form = TnrisForumTrainingForm
+    ordering = ('title',)
+    list_display = ('title', 'instructor', 'start_date_time', 'end_date_time')
+    search_fields = ('title', 'instructor')

src/data_hub/tnris_org/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class TnrisOrgConfig(AppConfig):
+    name = 'tnris_org'
+    verbose_name = 'TNRIS.org'