From 14a88701f62cd8f543712b0b2983af30510814b3 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 22 Jan 2020 11:52:32 -0600 Subject: [PATCH 1/6] update database utility function lambda --- .gitignore | 4 ++ .../lambda_function.py | 42 +++++++++++++++++++ .../requirements.txt | 2 + 3 files changed, 48 insertions(+) create mode 100644 lambda-update_database_record_utility/lambda_function.py create mode 100644 lambda-update_database_record_utility/requirements.txt diff --git a/.gitignore b/.gitignore index 9eac56e3..132be4c4 100644 --- a/.gitignore +++ b/.gitignore @@ -178,6 +178,10 @@ lambda-refresh_materialized_views/* !lambda-refresh_materialized_views/lambda_function.py !lambda-refresh_materialized_views/requirements.txt +lambda-update_database_record_utility/* +!lambda-update_database_record_utility/lambda_function.py +!lambda-update_database_record_utility/requirements.txt + vault-password\.txt .vault/ diff --git a/lambda-update_database_record_utility/lambda_function.py b/lambda-update_database_record_utility/lambda_function.py new file mode 100644 index 00000000..4e405e13 --- /dev/null +++ b/lambda-update_database_record_utility/lambda_function.py @@ -0,0 +1,42 @@ +# --------------- IMPORTS --------------- +import os, psycopg2 + +# Database Connection Info +database = os.environ.get('DB_NAME') +username = os.environ.get('DB_USER') +password = os.environ.get('DB_PASSWORD') +host = os.environ.get('DB_HOST') +port = os.environ.get('DB_PORT') + +conn_string = "dbname='%s' user='%s' host='%s' password='%s' port='%s'" % (database, username, host, password, port) + +# --------------- Main handler ------------------ + +def lambda_handler(event, context): + print(event) + table = event['table'] + field = event['field'] + value = event['value'] + where = event['where'] + # connect to database + conn = psycopg2.connect(conn_string) + cur = conn.cursor() + + if isinstance(value, str): + value = "'" + value + "'" + + q = "UPDATE %s SET %s = %s;" % (table, field, value) + + if where in event.keys() and where != '': + q = q.replace(";", " WHERE %s;" % where) + + print(q) + cur.execute(q) + conn.commit() + + cur.close() + conn.close() + print("that's all folks!!") + +if __name__ == '__main__': + lambda_handler(event='event', context='context') diff --git a/lambda-update_database_record_utility/requirements.txt b/lambda-update_database_record_utility/requirements.txt new file mode 100644 index 00000000..0b0ab7c0 --- /dev/null +++ b/lambda-update_database_record_utility/requirements.txt @@ -0,0 +1,2 @@ +psycopg2==2.7.7 +psycopg2-binary==2.7.7 From 582da28c689afd7ee69d814a09b694d3fdda2016 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Fri, 24 Jan 2020 09:06:57 -0600 Subject: [PATCH 2/6] where key event fix --- lambda-update_database_record_utility/lambda_function.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lambda-update_database_record_utility/lambda_function.py b/lambda-update_database_record_utility/lambda_function.py index 4e405e13..fb194158 100644 --- a/lambda-update_database_record_utility/lambda_function.py +++ b/lambda-update_database_record_utility/lambda_function.py @@ -17,7 +17,10 @@ def lambda_handler(event, context): table = event['table'] field = event['field'] value = event['value'] - where = event['where'] + try: + where = event['where'] + except: + print('No "where" key in event!!!') # connect to database conn = psycopg2.connect(conn_string) cur = conn.cursor() @@ -27,7 +30,7 @@ def lambda_handler(event, context): q = "UPDATE %s SET %s = %s;" % (table, field, value) - if where in event.keys() and where != '': + if 'where' in event.keys() and event['where'] != '': q = q.replace(";", " WHERE %s;" % where) print(q) From 2a3aa1b61094ca21b60e4eff1aa0df6f513c07f6 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Fri, 24 Jan 2020 09:11:35 -0600 Subject: [PATCH 3/6] handle potential error if no where argument in lambda event --- lambda-update_database_record_utility/lambda_function.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambda-update_database_record_utility/lambda_function.py b/lambda-update_database_record_utility/lambda_function.py index fb194158..c9d82919 100644 --- a/lambda-update_database_record_utility/lambda_function.py +++ b/lambda-update_database_record_utility/lambda_function.py @@ -21,6 +21,7 @@ def lambda_handler(event, context): where = event['where'] except: print('No "where" key in event!!!') + event['where'] = None # connect to database conn = psycopg2.connect(conn_string) cur = conn.cursor() @@ -30,7 +31,7 @@ def lambda_handler(event, context): q = "UPDATE %s SET %s = %s;" % (table, field, value) - if 'where' in event.keys() and event['where'] != '': + if event['where'] != None: q = q.replace(";", " WHERE %s;" % where) print(q) From 09e683081c373730ee0b0722de052399cd0fda3c Mon Sep 17 00:00:00 2001 From: John Haney Date: Wed, 29 Jan 2020 10:57:28 -0600 Subject: [PATCH 4/6] Feature/tnris org documents tiny preview bug (#65) * removed tiny_preview from tnris_org documents issue #64 * image and document form tweaking; helper text * admin.py and forms.py tnris_org - pretty up htmlwith divs and margin/style so no overlap, etc --- src/data_hub/tnris_org/admin.py | 19 +++---------------- src/data_hub/tnris_org/forms.py | 8 ++++---- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/data_hub/tnris_org/admin.py b/src/data_hub/tnris_org/admin.py index e04214de..c88435d4 100644 --- a/src/data_hub/tnris_org/admin.py +++ b/src/data_hub/tnris_org/admin.py @@ -52,7 +52,7 @@ def image_url_link(self, obj): """.replace('{id}', htmlId) js = format_html(js) return format_html( - u'{0}COPY URL', + u'{0}
COPY URL
', js, htmlId, htmlId, @@ -72,7 +72,7 @@ class TnrisDocumentAdmin(admin.ModelAdmin): model = TnrisDocument form = DocumentForm ordering = ('document_name',) - list_display = ('document_name', 'document_url_link', 'tiny_preview', 'created') + list_display = ('document_name', 'document_url_link', 'created') search_fields = ('document_name', 'document_url') def document_url_link(self, obj): @@ -89,26 +89,13 @@ def document_url_link(self, obj): """.replace('{id}', htmlId) js = format_html(js) return format_html( - u'{0}COPY URL', + u'{0}
COPY URL
', js, htmlId, htmlId, obj.document_url ) - def tiny_preview(self, obj): - # tiny preview of .pdf files errors/breaks on page load when src is cached. - # this means all tiny previews display on hard reload, but .pdfs don't on - # normal reload. so, we append a current datetime query string to the request - # so the browser recognizes each load as a new request and doesn't load the - # cached version. - no_cache = str(datetime.today()) - return format_html( - u'', - obj.document_url, - no_cache - ) - @admin.register(TnrisTraining) class TnrisTrainingAdmin(admin.ModelAdmin): diff --git a/src/data_hub/tnris_org/forms.py b/src/data_hub/tnris_org/forms.py index 3784ef20..88c82b3b 100644 --- a/src/data_hub/tnris_org/forms.py +++ b/src/data_hub/tnris_org/forms.py @@ -32,7 +32,7 @@ def render(self, name, value, attrs=None): if value is None: html = Template("""""") else: - html = Template("""{0}COPY URL
""".format(js)) + html = Template("""{0}
COPY URL
""".format(js)) return mark_safe(html.substitute(link=value,name=name)) @@ -51,7 +51,7 @@ def render(self, name, value, attrs=None): if value is None: html = Template("""""") else: - html = Template("""{0}COPY URL
""".format(js)) + html = Template("""{0}
COPY URL
""".format(js)) return mark_safe(html.substitute(link=value,name=name)) @@ -69,7 +69,7 @@ class Meta: model = TnrisImage fields = ('__all__') - image_url = forms.FileField(required=False, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record.") + image_url = forms.FileField(required=False, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record. The best method to overwrite would be to delete the existing file and re-upload a new file with the same name.") # boto3 s3 object client = boto3.client('s3') @@ -128,7 +128,7 @@ class Meta: model = TnrisDocument fields = ('__all__') - document_url = forms.FileField(required=False, widget=DocumentWidget, help_text="Choose a document file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record.") + document_url = forms.FileField(required=False, widget=DocumentWidget, help_text="Choose a document file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record. The best method to overwrite would be to delete the existing file and re-upload a new file with the same name.") # boto3 s3 object client = boto3.client('s3') From e1f77929c7f4a43da5205ef774fa28817966f337 Mon Sep 17 00:00:00 2001 From: John Haney Date: Wed, 29 Jan 2020 15:23:35 -0600 Subject: [PATCH 5/6] Feature/map downloads bug (#66) * issue #63; disable choose file input if value already exists * remove "Current" as it is not needed. * disable msd map downloads file input for existing docs/urls; added copy function instead resolves issue #63 --- src/data_hub/msd/forms.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/data_hub/msd/forms.py b/src/data_hub/msd/forms.py index dff84993..c4afd287 100644 --- a/src/data_hub/msd/forms.py +++ b/src/data_hub/msd/forms.py @@ -1,14 +1,22 @@ from django import forms from django.core.exceptions import ValidationError -# from django.db.utils import ProgrammingError - -from .models import MapCollection, MapDataRelate, MapDownload, MapSize, PixelsPerInch -from lcd.models import Collection from string import Template from django.utils.safestring import mark_safe -import boto3, os +from django.db.utils import ProgrammingError +from .models import ( + MapCollection, + MapDataRelate, + MapDownload, + MapSize, + PixelsPerInch +) + +import os +import boto3 + +from lcd.models import Collection class PictureWidget(forms.widgets.Widget): @@ -19,7 +27,20 @@ def render(self, name, value, attrs=None): class PdfWidget(forms.widgets.Widget): def render(self, name, value, attrs=None): - html = Template("""""") + js = """ + + """ + + if value: + html = Template("""{0}
""".format(js)) + else: + html = Template("""""") return mark_safe(html.substitute(link=value,name=name)) @@ -33,7 +54,7 @@ class Meta: 'label' ) - download_url = forms.FileField(required=False, widget=PdfWidget, help_text="Upload map download file. PDF is recommended. 75MB max size.") + download_url = forms.FileField(required=False, widget=PdfWidget, help_text="Upload map download file. PDF is recommended. 75MB max size. Overwriting files is not allowed. Delete the record and create a new one with the new file if you are attempting to overwrite.") # FILE UPLOAD FOR DOWNLOAD FILE (PDF SUGGESTED) # boto3 s3 object From 81de6c5c23aa4cb96991f4089c0478c26a57f9c4 Mon Sep 17 00:00:00 2001 From: John Haney Date: Fri, 31 Jan 2020 14:28:55 -0600 Subject: [PATCH 6/6] Feature/tnris org docs sgm notes (#68) * new sgm_note field added to TnrisDocument model - boolean issue #62; serializers, viewsets, urls, models, forms edited; now available in api with endpoint - api/v1/tnris_org/sgm_note * comments and additional helper text added issue #62 --- src/data_hub/tnris_org/forms.py | 1 + .../migrations/0024_tnrisdocument_sgm_note.py | 18 +++++++++++++++ src/data_hub/tnris_org/models.py | 5 ++++ src/data_hub/tnris_org/serializers.py | 14 ++++++++++- src/data_hub/tnris_org/urls.py | 2 ++ src/data_hub/tnris_org/viewsets.py | 23 ++++++++++++++++--- 6 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 src/data_hub/tnris_org/migrations/0024_tnrisdocument_sgm_note.py diff --git a/src/data_hub/tnris_org/forms.py b/src/data_hub/tnris_org/forms.py index 88c82b3b..57da8329 100644 --- a/src/data_hub/tnris_org/forms.py +++ b/src/data_hub/tnris_org/forms.py @@ -129,6 +129,7 @@ class Meta: fields = ('__all__') document_url = forms.FileField(required=False, widget=DocumentWidget, help_text="Choose a document file and 'Save' this form to upload & save it to the database. Attempting to overwrite with a new file will only create a new record. The best method to overwrite would be to delete the existing file and re-upload a new file with the same name.") + sgm_note = forms.BooleanField(required=False, label="GIS Solutions Group Notes", help_text="Check this box to identify this as a Solutions Group Notes document. This is required to view the document on tnris.org. Be sure to name the file correctly - 'YYYY-MM-DD-GIS-SG-Meeting-Notes.pdf'. The file name is important for the order these documents are presented on tnris.org.") # boto3 s3 object client = boto3.client('s3') diff --git a/src/data_hub/tnris_org/migrations/0024_tnrisdocument_sgm_note.py b/src/data_hub/tnris_org/migrations/0024_tnrisdocument_sgm_note.py new file mode 100644 index 00000000..31fe1605 --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0024_tnrisdocument_sgm_note.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2020-01-31 17:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0023_tnrisgiocalendarevent_community_meeting_agenda_url'), + ] + + operations = [ + migrations.AddField( + model_name='tnrisdocument', + name='sgm_note', + field=models.BooleanField(default=False, verbose_name='Solutions Group Note'), + ), + ] diff --git a/src/data_hub/tnris_org/models.py b/src/data_hub/tnris_org/models.py index 4a8b128f..de33ce9d 100644 --- a/src/data_hub/tnris_org/models.py +++ b/src/data_hub/tnris_org/models.py @@ -80,6 +80,11 @@ class Meta: 'Document URL', max_length=255, ) + sgm_note = models.BooleanField( + 'Solutions Group Note', + default=False, + null=False + ) created = models.DateTimeField( 'Created', auto_now_add=True diff --git a/src/data_hub/tnris_org/serializers.py b/src/data_hub/tnris_org/serializers.py index 95dd73cf..9252a0c5 100644 --- a/src/data_hub/tnris_org/serializers.py +++ b/src/data_hub/tnris_org/serializers.py @@ -5,7 +5,8 @@ TnrisForumTraining, TnrisInstructorType, CompleteForumTrainingView, - TnrisGioCalendarEvent + TnrisGioCalendarEvent, + TnrisDocument ) from datetime import datetime @@ -148,3 +149,14 @@ def get_pretty_time(self, obj): pt = "%s-%s" % (st.strftime('%I:%M%p').lstrip("0"), et.strftime('%I:%M%p').lstrip("0")) return pt + + +class TnrisSGMDocumentSerializer(serializers.ModelSerializer): + class Meta: + model = TnrisDocument + fields = ('document_id', + 'document_name', + 'document_url', + 'sgm_note', + 'created', + 'last_modified',) diff --git a/src/data_hub/tnris_org/urls.py b/src/data_hub/tnris_org/urls.py index 895509f4..f0063ffa 100644 --- a/src/data_hub/tnris_org/urls.py +++ b/src/data_hub/tnris_org/urls.py @@ -24,6 +24,7 @@ TnrisInstructorTypeViewSet, CompleteForumTrainingViewSet, TnrisGioCalendarEventViewSet, + TnrisSGMDocumentViewSet, ) router = routers.DefaultRouter(trailing_slash=False) @@ -32,6 +33,7 @@ router.register(r'complete_forum_training/?', CompleteForumTrainingViewSet, base_name="CompleteForumTrainingView") router.register(r'instructor_type/?', TnrisInstructorTypeViewSet, base_name="TnrisInstructorType") router.register(r'gio_calendar/?', TnrisGioCalendarEventViewSet, base_name="TnrisGioCalendarEvent") +router.register(r'sgm_note/?', TnrisSGMDocumentViewSet, base_name="TnrisDocument") schema_view = get_swagger_view(title='TNRIS.org API') diff --git a/src/data_hub/tnris_org/viewsets.py b/src/data_hub/tnris_org/viewsets.py index 255172e5..afb4e12d 100644 --- a/src/data_hub/tnris_org/viewsets.py +++ b/src/data_hub/tnris_org/viewsets.py @@ -7,14 +7,16 @@ TnrisForumTraining, TnrisInstructorType, CompleteForumTrainingView, - TnrisGioCalendarEvent + TnrisGioCalendarEvent, + TnrisDocument ) from .serializers import ( TnrisTrainingSerializer, TnrisForumTrainingSerializer, TnrisInstructorTypeSerializer, CompleteForumTrainingViewSerializer, - TnrisGioCalendarEventSerializer + TnrisGioCalendarEventSerializer, + TnrisSGMDocumentSerializer ) @@ -130,4 +132,19 @@ def get_queryset(self): args[field] = value # get records using query. order by chronological start queryset = TnrisGioCalendarEvent.objects.filter(**args).order_by('start_date', 'start_time') - return queryset \ No newline at end of file + return queryset + + +class TnrisSGMDocumentViewSet(viewsets.ReadOnlyModelViewSet): + """ + Retrieve all Solutions Group Meeting Documents for tnris.org frontend + """ + serializer_class = TnrisSGMDocumentSerializer + http_method_names = ['get'] + + def get_queryset(self): + # get records using query + args = {'sgm_note': True} + # order by document file name + queryset = TnrisDocument.objects.filter(**args).order_by('document_name') + return queryset