From 2496da7a1237b0670471c40eaef04b0a503e1290 Mon Sep 17 00:00:00 2001 From: Tom Z Date: Wed, 12 Feb 2020 18:21:37 -0500 Subject: [PATCH 01/71] issue-3286, beginning of integratin with inaturalist --- .../treemap/js/src/lib/socialMediaSharing.js | 64 ++++++ ...inaturalistobservation_inaturalistphoto.py | 34 +++ opentreemap/treemap/models.py | 14 ++ opentreemap/treemap/routes.py | 9 + .../templates/treemap/map_feature_detail.html | 2 +- .../partials/photo_social_media_sharing.html | 8 +- opentreemap/treemap/urls.py | 6 + opentreemap/treemap/views/map_feature.py | 201 +++++++++++++++++- 8 files changed, 334 insertions(+), 4 deletions(-) create mode 100644 opentreemap/treemap/migrations/0047_inaturalistobservation_inaturalistphoto.py diff --git a/opentreemap/treemap/js/src/lib/socialMediaSharing.js b/opentreemap/treemap/js/src/lib/socialMediaSharing.js index 769d4f510..8d937d04f 100644 --- a/opentreemap/treemap/js/src/lib/socialMediaSharing.js +++ b/opentreemap/treemap/js/src/lib/socialMediaSharing.js @@ -8,10 +8,19 @@ var $ = require('jquery'), Bacon = require('baconjs'), R = require('ramda'), _ = require('lodash'), + config = require('treemap/lib/config.js'), _DONT_SHOW_AGAIN_KEY = 'social-media-sharing-dont-show-again', _SHARE_CONTAINER_SIZE = 300, + // for iNaturalist + _APP_ID = 'db6db69ef86d5a21a4c9876bcaebad059db3b1ed90f30255c6d9e8bdaebf0513', + + photoInfo = { + PhotoDetailUrl: '', + PhotoUrl: '' + }, + attrs = { dataUrlTemplate: 'data-url-template', dataClass: 'data-class', @@ -22,6 +31,8 @@ var $ = require('jquery'), dom = { dontShowAgain: '[' + attrs.dataClass + '="' + _DONT_SHOW_AGAIN_KEY + '"]', + loginToINaturalist: '[' + attrs.dataClass + '="social-media-login-to-inaturalist"]', + submitToINaturalist: '[' + attrs.dataClass + '="social-media-submit-to-inaturalist"]', photoModal: '#social-media-sharing-photo-upload-modal', photoPreview: '#social-media-sharing-photo-upload-preview', shareLinkSelector: '[' + attrs.dataUrlTemplate + ']', @@ -62,6 +73,9 @@ function renderPhotoModal (imageData) { photoUrl = $photo.attr(attrs.mapFeaturePhotoImageAbsoluteUrl), $photoPreview = $(dom.photoPreview); + photoInfo.PhotoDetailUrl = photoDetailUrl; + photoInfo.PhotoUrl = photoUrl; + // Validation errors (image invalid, image too big) are only returned as DOM // elements. In order to skip showing the share dialog we need to check the // dialog markup for the error message element. @@ -73,6 +87,54 @@ function renderPhotoModal (imageData) { _.each($anchors, generateHref(photoDetailUrl, photoUrl)); } +function loginToINaturalist(e) { + // run the auth + var site = "https://www.inaturalist.org"; + var redirectUri = "http://localhost:7070/jerseycity/inaturalist/"; + /* + // For PKCE workflow, just in the client side + var codeVerifier = 'test'; + var codeChallenge = window.btoa(codeVerifier); + var url = `${site}/oauth/authorize?client_id=${_APP_ID}&redirect_uri=${redirectUri}&response_type=code&code_challenge_method=S256&code_challenge=${codeChallenge}` + */ + var url = `${site}/oauth/authorize?client_id=${_APP_ID}&redirect_uri=${redirectUri}&response_type=code` + + window.location.href = url; +} + +function submitToINaturalist(e) { + var featureId = window.otm.mapFeature.featureId, + inaturalistUrl = '/jerseycity/inaturalist-add/'; + + var data = { + 'photoDetailUrl': photoInfo.PhotoDetailUrl, + 'photoUrl': photoInfo.PhotoUrl, + 'featureId': featureId + }; + console.log(data); + + $.ajax({ + url: inaturalistUrl, + type: 'POST', + contentType: "application/json", + data: JSON.stringify(data), + success: onSuccess, + error: onError + }); + + console.log("here"); +} + +function onSuccess(result) { + console.log('success'); + console.log(result); +} + +function onError(result) { + console.log('error'); + console.log(result); +} + module.exports.init = function(options) { var imageFinishedStream = options.imageFinishedStream || Bacon.never(); $(dom.toggle).on('click', function () { @@ -84,4 +146,6 @@ module.exports.init = function(options) { .onValue(renderPhotoModal); $(dom.dontShowAgain).on('click', setDontShowAgainVal); + $(dom.loginToINaturalist).on('click', loginToINaturalist); + $(dom.submitToINaturalist).on('click', submitToINaturalist); }; diff --git a/opentreemap/treemap/migrations/0047_inaturalistobservation_inaturalistphoto.py b/opentreemap/treemap/migrations/0047_inaturalistobservation_inaturalistphoto.py new file mode 100644 index 000000000..bf0db6f5c --- /dev/null +++ b/opentreemap/treemap/migrations/0047_inaturalistobservation_inaturalistphoto.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2020-01-11 17:37 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('treemap', '0046_auto_20170907_0937'), + ] + + operations = [ + migrations.CreateModel( + name='INaturalistObservation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('observation_id', models.IntegerField()), + ('map_feature', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='treemap.MapFeature')), + ('tree', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='treemap.Tree')), + ], + ), + migrations.CreateModel( + name='INaturalistPhoto', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('inaturalist_photo_id', models.IntegerField()), + ('observation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='treemap.INaturalistObservation')), + ('tree_photo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='treemap.TreePhoto')), + ], + ), + ] diff --git a/opentreemap/treemap/models.py b/opentreemap/treemap/models.py index afd65ce06..50469e453 100644 --- a/opentreemap/treemap/models.py +++ b/opentreemap/treemap/models.py @@ -1506,3 +1506,17 @@ class Meta: def __init__(self, *args, **kwargs): super(ITreeCodeOverride, self).__init__(*args, **kwargs) self.populate_previous_state() + + +class INaturalistObservation(models.Model): + # this is the observation_id from iNaturalist + observation_id = models.IntegerField() + map_feature = models.ForeignKey(MapFeature) + tree = models.ForeignKey(Tree) + + +class INaturalistPhoto(models.Model): + tree_photo = models.ForeignKey(TreePhoto) + observation = models.ForeignKey(INaturalistObservation) + inaturalist_photo_id = models.IntegerField() + diff --git a/opentreemap/treemap/routes.py b/opentreemap/treemap/routes.py index 99f9e22a8..9b59f6988 100644 --- a/opentreemap/treemap/routes.py +++ b/opentreemap/treemap/routes.py @@ -279,3 +279,12 @@ json_api_call, return_400_if_validation_errors, user_views.users) + + +### INATURALIST +inaturalist = feature_views.inaturalist +inaturalist_add = do( + instance_request, + require_http_method('POST'), + json_api_call, + feature_views.inaturalist_add) diff --git a/opentreemap/treemap/templates/treemap/map_feature_detail.html b/opentreemap/treemap/templates/treemap/map_feature_detail.html index 808f5c99b..5752f871f 100644 --- a/opentreemap/treemap/templates/treemap/map_feature_detail.html +++ b/opentreemap/treemap/templates/treemap/map_feature_detail.html @@ -29,8 +29,8 @@ {% trans "Add a Photo" as upload_title %} {% include "treemap/partials/upload_file.html" with title=upload_title upload_url=upload_photo_endpoint %} + {% include "treemap/partials/photo_social_media_sharing.html" %} {% if request.instance.is_public %} - {% include "treemap/partials/photo_social_media_sharing.html" %} {% endif %} {# Modal for viewing and rotating photos #} diff --git a/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html b/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html index 84ef93bd2..7538b2b10 100644 --- a/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html +++ b/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html @@ -24,7 +24,13 @@

{% trans "Photo Added!" %}

target="_blank"> diff --git a/opentreemap/treemap/urls.py b/opentreemap/treemap/urls.py index a148d4a41..561efe491 100644 --- a/opentreemap/treemap/urls.py +++ b/opentreemap/treemap/urls.py @@ -78,4 +78,10 @@ url(r'^users/%s/edits/$' % USERNAME_PATTERN, routes.instance_user_audits), url(r'^users/$', routes.users, name="users"), + + url(r'^inaturalist/$', + routes.inaturalist, name='inaturalist'), + + url(r'^inaturalist-add/$', + routes.inaturalist_add, name='inaturalist_add'), ] diff --git a/opentreemap/treemap/views/map_feature.py b/opentreemap/treemap/views/map_feature.py index b3cd7f007..861a2b20e 100644 --- a/opentreemap/treemap/views/map_feature.py +++ b/opentreemap/treemap/views/map_feature.py @@ -3,11 +3,14 @@ from __future__ import unicode_literals from __future__ import division +import datetime import json import hashlib +import re +import requests from functools import wraps -from django.http import HttpResponse +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404, render from django.core.exceptions import ValidationError from django.conf import settings @@ -22,7 +25,8 @@ from treemap.units import Convertible from treemap.models import (Tree, Species, MapFeature, - MapFeaturePhoto, TreePhoto, Favorite) + MapFeaturePhoto, TreePhoto, Favorite, + INaturalistPhoto, INaturalistObservation) from treemap.util import (package_field_errors, to_object_name) from treemap.images import get_image_from_request @@ -450,3 +454,196 @@ def unfavorite_map_feature(request, instance, feature_id): Favorite.objects.filter(user=request.user, map_feature=feature).delete() return {'success': True} + + +def inaturalist(request, *args, **kwargs): + """ + """ + base_url = "https://www.inaturalist.org" + + app_id = 'db6db69ef86d5a21a4c9876bcaebad059db3b1ed90f30255c6d9e8bdaebf0513'; + app_secret = '19bcad3978b02e2f79fc7723bbd295a3c95f60f6246b99a3f751f21407bd6095'; + code = request.GET['code'] + redirect_uri = request.META['HTTP_REFERER'] + + # to build up the redirect url + #request.path + #request.is_secure() + #request.get_host() + + payload = { + 'client_id': app_id, + 'client_secret': app_secret, + 'code': code, + 'redirect_uri': 'http://localhost:7070/jerseycity/inaturalist/', + 'grant_type': 'authorization_code' + } + + r = requests.post( + url="{base_url}/oauth/token".format(base_url=base_url), + data=payload + ) + request.session['inaturalist_token'] = r.json()['access_token'] + import ipdb; ipdb.set_trace() # BREAKPOINT + + return HttpResponseRedirect(redirect_uri) + + +def get_photo_id_from_photo_detail_url(url, feature_id): + """ + """ + return int(re.match(r'.*/{}/photo/(\d+)/detail'.format(feature_id), url).groups()[0]) + + +def create_observation(token, latitude, longitude): + """ + """ + base_url = "https://www.inaturalist.org" + headers = {'Authorization': 'Bearer {}'.format(token)} + params = {'observation': { + 'observed_on_string': datetime.datetime.now().isoformat(), + 'latitude': latitude, + 'longitude': longitude + } + } + + response = requests.post( + url="{base_url}/observations.json".format(base_url=base_url), + json=params, + headers=headers + ) + ''' + return { + u'cached_votes_total': 0, + u'captive': False, + u'comments_count': 0, + u'community_taxon_id': None, + u'created_at': u'2020-01-07T22:04:44.757-05:00', + u'created_at_utc': u'2020-01-08T03:04:44.757Z', + u'delta': False, + u'description': None, + u'faves_count': 0, + u'geoprivacy': None, + u'iconic_taxon_id': None, + u'iconic_taxon_name': None, + u'id': 37388076, + u'id_please': False, + u'identifications_count': 0, + u'last_indexed_at': u'2020-01-07T19:04:49.322-08:00', + u'latitude': u'40.7083055556', + u'license': u'CC-BY-NC', + u'location_is_exact': False, + u'longitude': u'-74.0893888889', + u'map_scale': None, + u'mappable': True, + u'num_identification_agreements': 0, + u'num_identification_disagreements': 0, + u'oauth_application_id': 385, + u'observation_photos_count': 0, + u'observation_sounds_count': 0, + u'observed_on': u'2020-01-07', + u'observed_on_string': u'2020-01-07T21:04:44.393925', + u'old_uuid': None, + u'out_of_range': None, + u'owners_identification_from_vision': None, + u'place_guess': u'Hudson County, US-NJ, US', + u'positional_accuracy': None, + u'positioning_device': None, + u'positioning_method': None, + u'private_latitude': None, + u'private_longitude': None, + u'private_place_guess': None, + u'private_positional_accuracy': None, + u'project_observations': [], + u'public_positional_accuracy': None, + u'quality_grade': u'casual', + u'site_id': 1, + u'species_guess': None, + u'taxon_geoprivacy': None, + u'taxon_id': None, + u'time_observed_at': u'2020-01-07T21:04:44.000-05:00', + u'time_observed_at_utc': u'2020-01-08T02:04:44.000Z', + u'time_zone': u'America/New_York', + u'timeframe': None, + u'updated_at': u'2020-01-07T22:04:44.757-05:00', + u'updated_at_utc': u'2020-01-08T03:04:44.757Z', + u'uri': None, + u'user_id': 2384052, + u'user_login': u'tzinckgraf', + u'uuid': u'0e26bafc-bd23-48d0-9bda-806450093c88', + u'zic_time_zone': None + } + ''' + return response.json()[0] + + +def add_photo_to_observation(token, observation_id, photo): + base_url = "https://www.inaturalist.org" + headers = {'Authorization': 'Bearer {}'.format(token)} + data = {'observation_photo[observation_id]': observation_id} + file_data = {'file': photo.image.file.file} + + response = requests.post( + url="{base_url}/observation_photos".format(base_url=base_url), + headers=headers, + data=data, + files=file_data + ) + ''' + return { + u'created_at': u'2020-01-07T22:17:04.531-05:00', + u'created_at_utc': u'2020-01-08T03:17:04.531Z', + u'id': 54883602, + u'observation_id': 37388076, + u'old_uuid': None, + u'photo': {u'attribution': u'(c) Thomas Zinckgraf, some rights reserved (CC BY-NC)', + u'created_at': u'2020-01-07T22:17:02.709-05:00', + u'id': 59263063, + u'large_url': None, + u'license': 2, + u'license_name': u'Creative Commons Attribution-NonCommercial License', + u'license_url': u'http://creativecommons.org/licenses/by-nc/4.0/', + u'medium_url': None, + u'native_original_image_url': None, + u'native_page_url': None, + u'native_photo_id': u'59263063', + u'native_realname': u'Thomas Zinckgraf', + u'native_username': u'tzinckgraf', + u'small_url': None, + u'square_url': None, + u'subtype': None, + u'thumb_url': None, + u'type': u'LocalPhoto', + u'updated_at': u'2020-01-07T22:17:02.709-05:00', + u'user_id': 2384052}, + u'photo_id': 59263063, + u'position': None, + u'updated_at': u'2020-01-07T22:17:04.531-05:00', + u'updated_at_utc': u'2020-01-08T03:17:04.531Z', + u'uuid': u'db6b5a49-eb92-4b12-8d3f-83f388ac55f0'} + ''' + + +def inaturalist_add(request, instance, *args, **kwargs): + try: + token = request.session['inaturalist_token'] + except KeyError: + return {'success': False} + + # INaturalistPhoto, INaturalistObservation + + body = json.loads(request.body) + feature_id = body['featureId'] + feature = get_map_feature_or_404(feature_id, instance) + tree = feature.safe_get_current_tree() + photo_id = get_photo_id_from_photo_detail_url(body['photoDetailUrl'], feature_id) + photo_class = TreePhoto if feature.is_plot else MapFeaturePhoto + photo = get_object_or_404(photo_class, pk=photo_id, map_feature=feature) + + (longitude, latitude) = feature.latlon.coords + + observation = create_observation(token, latitude, longitude) + photo_info = add_photo_to_observation(token, observation['id'], photo) + + import ipdb; ipdb.set_trace() # BREAKPOINT + return {'success': True} From 6202ceecbaa0820bd92495a705fca5380c9a31a2 Mon Sep 17 00:00:00 2001 From: Tom Z Date: Tue, 25 Feb 2020 22:41:31 -0500 Subject: [PATCH 02/71] issue-3287 - the start to adding labels --- .../css/sass/partials/pages/_treedetails.scss | 15 +++++ .../treemap/js/src/lib/imageLightbox.js | 35 +++++++++++- .../treemap/js/src/lib/socialMediaSharing.js | 57 ++++++++++++++++--- .../treemap/js/src/mapFeatureDetail.js | 5 +- opentreemap/treemap/lib/photo.py | 10 ++++ opentreemap/treemap/models.py | 12 ++++ opentreemap/treemap/routes.py | 18 ++++++ .../templates/treemap/map_feature_detail.html | 21 ++++++- .../treemap/partials/photo_carousel.html | 8 +++ .../partials/photo_social_media_sharing.html | 28 ++++----- opentreemap/treemap/urls.py | 2 + opentreemap/treemap/views/map_feature.py | 16 +++++- 12 files changed, 196 insertions(+), 31 deletions(-) diff --git a/assets/css/sass/partials/pages/_treedetails.scss b/assets/css/sass/partials/pages/_treedetails.scss index e9692883c..bfe981d43 100644 --- a/assets/css/sass/partials/pages/_treedetails.scss +++ b/assets/css/sass/partials/pages/_treedetails.scss @@ -200,6 +200,21 @@ display: block; opacity: .9; } + .photo-label { + position: absolute; + left: 6px; + top: 6px; + opacity: 0.85; + width: auto; + background: black; + color: white; + border: none; + transition: opacity 0.3s; + z-index: 999; + padding: 5px 12px; + font-size: 1.2rem !important; + border-radius: 6px; + } } } > a.carousel-control { diff --git a/opentreemap/treemap/js/src/lib/imageLightbox.js b/opentreemap/treemap/js/src/lib/imageLightbox.js index e453c053d..550ab9e43 100644 --- a/opentreemap/treemap/js/src/lib/imageLightbox.js +++ b/opentreemap/treemap/js/src/lib/imageLightbox.js @@ -91,7 +91,14 @@ module.exports.init = function(options) { endpoint = $endpointEl.attr('data-endpoint'), modeSelector = '[data-class="' + mode + '"]', notModeSelector = '[data-class]:not(' + modeSelector + ')', - $keepControl = $lightbox.find('[data-photo-keep]'); + $keepControl = $lightbox.find('[data-photo-keep]'), + + label = $endpointEl.attr('data-label'), + photoId = $endpointEl.attr('data-map-feature-photo-id'), + featureId = $endpointEl.attr('data-map-feature-id'), + labelSelector = '[data-class="label"]', + labelViewSelector = '.photo-label-view', + labelEditSelectSelector = '#photo-label'; $keepControl.off('click.delete-mode'); currentRotation = 0; @@ -104,6 +111,27 @@ module.exports.init = function(options) { $lightbox.find(notModeSelector).hide(); $lightbox.find('[data-photo-save]').attr('data-photo-save', endpoint); + var labelEl = $lightbox.find(labelSelector); + // set the label if it exists + if(label !== undefined && label !== "") { + console.log(mode); + var labelViewEl = $lightbox.find(labelViewSelector); + labelViewEl.html(label); + $lightbox.find(labelEditSelectSelector).val(label); + + if (mode === 'edit'){ + labelViewEl.hide(); + } else { + labelViewEl.show(); + } + + labelEl.show(); + + } else { + labelEl.hide(); + } + + // tzinckgraf if (1 === $deleteToggleEl.length) { $lightbox.find('[data-photo-confirm]').attr('data-photo-confirm', endpoint); $lightbox.find('[data-class="delete"] button').prop('disabled', false); @@ -124,6 +152,11 @@ module.exports.init = function(options) { $lightbox.find('[data-class="edit"]').show(); }); + $lightbox.on('click', '[data-photo-save-cancel]', function() { + $lightbox.find('[data-class]:not([data-class="view"])').hide(); + $lightbox.find('[data-class="view"]').show(); + }); + $lightbox.on('click', '[data-photo-rotate]', function(e) { var $target = $(e.currentTarget), $saveButton = $target diff --git a/opentreemap/treemap/js/src/lib/socialMediaSharing.js b/opentreemap/treemap/js/src/lib/socialMediaSharing.js index 769d4f510..4700217a5 100644 --- a/opentreemap/treemap/js/src/lib/socialMediaSharing.js +++ b/opentreemap/treemap/js/src/lib/socialMediaSharing.js @@ -6,8 +6,11 @@ var $ = require('jquery'), Bacon = require('baconjs'), + BU = require('treemap/lib/baconUtils.js'), R = require('ramda'), _ = require('lodash'), + reverse = require('reverse'), + config = require('treemap/lib/config.js'), _DONT_SHOW_AGAIN_KEY = 'social-media-sharing-dont-show-again', _SHARE_CONTAINER_SIZE = 300, @@ -15,19 +18,24 @@ var $ = require('jquery'), attrs = { dataUrlTemplate: 'data-url-template', dataClass: 'data-class', + mapFeatureId: 'data-map-feature-id', + mapFeaturePhotoId: 'data-map-feature-photo-id', mapFeaturePhotoDetailAbsoluteUrl: 'data-map-feature-photo-detail-absolute-url', mapFeaturePhotoImageAbsoluteUrl: 'data-map-feature-photo-image-absolute-url', - mapFeaturePhotoPreview: 'data-map-feature-photo-thumbnail' + mapFeaturePhotoPreview: 'data-map-feature-photo-thumbnail', }, dom = { dontShowAgain: '[' + attrs.dataClass + '="' + _DONT_SHOW_AGAIN_KEY + '"]', photoModal: '#social-media-sharing-photo-upload-modal', - photoPreview: '#social-media-sharing-photo-upload-preview', + photoPreview: '#label-photo-upload-preview', shareLinkSelector: '[' + attrs.dataUrlTemplate + ']', + mapFeatureId: '[' + attrs.mapFeatureId + ']', + mapFeaturePhotoId: '[' + attrs.mapFeaturePhotoId + ']', mapFeaturePhotoDetailAbsoluteUrl: '[' + attrs.mapFeaturePhotoDetailAbsoluteUrl + ']', toggle: '.share', - container: '.js-container' + container: '.js-container', + photoLabel: '#photo-label' }, generateHref = R.curry( @@ -60,17 +68,48 @@ function renderPhotoModal (imageData) { $photo = $carousel.find(dom.mapFeaturePhotoDetailAbsoluteUrl), photoDetailUrl = $photo.attr(attrs.mapFeaturePhotoDetailAbsoluteUrl), photoUrl = $photo.attr(attrs.mapFeaturePhotoImageAbsoluteUrl), + + mapFeatureId = $photo.attr(attrs.mapFeatureId), + mapFeaturePhotoId = $photo.attr(attrs.mapFeaturePhotoId), + $photoPreview = $(dom.photoPreview); // Validation errors (image invalid, image too big) are only returned as DOM // elements. In order to skip showing the share dialog we need to check the // dialog markup for the error message element. - if ($(imageData.data.result).filter('[data-photo-upload-failed]').length > 0) { - return; - } + //if ($(imageData.data.result).filter('[data-photo-upload-failed]').length > 0) { + // return; + //} $photoModal.modal('show'); $photoPreview.attr('src', $photo.attr(attrs.mapFeaturePhotoPreview)); _.each($anchors, generateHref(photoDetailUrl, photoUrl)); + + $(dom.photoLabel).on('change', function(e) { + console.log(e); + var value = e.target.value; + var url = reverse.map_feature_photo({ + instance_url_name: config.instance.url_name, + feature_id: window.otm.mapFeature.featureId, + photo_id: mapFeaturePhotoId + }) + '/label'; + + var stream = BU.jsonRequest('POST', url)({'label': value}); + stream.onValue(function() { + console.log("done"); + debugger; + }); + + /* + var addStream = $addUser + .asEventStream('click') + .map(function () { + return {'email': $addUserEmail.val()}; + }) + .flatMap(BU.jsonRequest('POST', url)); + */ + + debugger; + }); } module.exports.init = function(options) { @@ -79,9 +118,9 @@ module.exports.init = function(options) { $(dom.container).toggle(_SHARE_CONTAINER_SIZE); }); - imageFinishedStream - .filter(shouldShowSharingModal) - .onValue(renderPhotoModal); + imageFinishedStream.onValue(renderPhotoModal); + //.filter(shouldShowSharingModal) $(dom.dontShowAgain).on('click', setDontShowAgainVal); + }; diff --git a/opentreemap/treemap/js/src/mapFeatureDetail.js b/opentreemap/treemap/js/src/mapFeatureDetail.js index d745caa46..6b9d983ed 100644 --- a/opentreemap/treemap/js/src/mapFeatureDetail.js +++ b/opentreemap/treemap/js/src/mapFeatureDetail.js @@ -66,6 +66,7 @@ function init() { dataType: 'html' }); + // tzinckgraf imageLightbox.init({ imageFinishedStream: imageFinishedStream, imageContainer: '#photo-carousel', @@ -192,18 +193,20 @@ function init() { if (config.instance.basemap.type === 'google') { var $streetViewContainer = $(dom.streetView); $streetViewContainer.show(); + /* var panorama = streetView.create({ streetViewElem: $streetViewContainer[0], noStreetViewText: config.trans.noStreetViewText, location: window.otm.mapFeature.location.point }); + */ form.saveOkStream .onValue(function () { // If location is an array, we are editing a polygonal map // feature. The page triggers a full postback after editing a // polygon map feature. if (!_.isArray(currentMover.location)) { - panorama.update(currentMover.location); + //panorama.update(currentMover.location); } }); } diff --git a/opentreemap/treemap/lib/photo.py b/opentreemap/treemap/lib/photo.py index f1c1bea7d..592b1826e 100644 --- a/opentreemap/treemap/lib/photo.py +++ b/opentreemap/treemap/lib/photo.py @@ -34,6 +34,16 @@ def context_dict_for_photo(request, photo): photo_dict['thumbnail'] = thumbnail_url photo_dict['raw'] = photo + # add the label + # TODO use a OneToOne mapping + labels = photo.mapfeaturephotolabel_set.all() + if labels: + photo_dict['has_label'] = True + photo_dict['label'] = labels[0].name + photo_dict['label_id'] = labels[0].id + else: + photo_dict['has_label'] = False + url = reverse( 'map_feature_photo_detail', kwargs={'instance_url_name': photo.map_feature.instance.url_name, diff --git a/opentreemap/treemap/models.py b/opentreemap/treemap/models.py index afd65ce06..78cb87c41 100644 --- a/opentreemap/treemap/models.py +++ b/opentreemap/treemap/models.py @@ -1506,3 +1506,15 @@ class Meta: def __init__(self, *args, **kwargs): super(ITreeCodeOverride, self).__init__(*args, **kwargs) self.populate_previous_state() + + +class MapFeaturePhotoLabel(models.Model): + """ + Provide a tag for a phot + """ + map_feature_photo = models.ForeignKey(MapFeaturePhoto) + name = models.CharField(max_length=40) + + class Meta: + unique_together = ('map_feature_photo', 'name') + diff --git a/opentreemap/treemap/routes.py b/opentreemap/treemap/routes.py index 99f9e22a8..35b9535ad 100644 --- a/opentreemap/treemap/routes.py +++ b/opentreemap/treemap/routes.py @@ -173,6 +173,24 @@ POST=add_map_feature_photo_do(feature_views.rotate_map_feature_photo), DELETE=delete_photo) +add_map_feature_photo_label = add_map_feature_photo_do( + feature_views.add_map_feature_photo_label) + +add_map_feature_photo_do = partial( + do, + require_http_method("POST"), + login_or_401, + instance_request, + creates_instance_user, + render_template('treemap/partials/photo_carousel.html')) + +map_feature_photo_detail = do( + instance_request, + require_http_method('GET'), + render_template('treemap/map_feature_photo_detail.html'), + feature_views.map_feature_photo_detail) + +# tzinckgraf map_feature_photo_detail = do( instance_request, require_http_method('GET'), diff --git a/opentreemap/treemap/templates/treemap/map_feature_detail.html b/opentreemap/treemap/templates/treemap/map_feature_detail.html index 808f5c99b..df43887e5 100644 --- a/opentreemap/treemap/templates/treemap/map_feature_detail.html +++ b/opentreemap/treemap/templates/treemap/map_feature_detail.html @@ -53,9 +53,26 @@
+ + {# Button and dropdown for label, depending on edit or view #} +
+ + +
+
+ + +
+ +
-
diff --git a/opentreemap/treemap/templates/treemap/partials/photo_carousel.html b/opentreemap/treemap/templates/treemap/partials/photo_carousel.html index dd8fb235b..e8d667ecc 100644 --- a/opentreemap/treemap/templates/treemap/partials/photo_carousel.html +++ b/opentreemap/treemap/templates/treemap/partials/photo_carousel.html @@ -10,6 +10,9 @@ data-photo-src="{{ photo.image }}" {% if forloop.first %} {# this is the *actual* photo for the map feature and will be used by the sharing code when a new upload takes place #} + data-map-feature-photo-id="{{ photo.id }}" + data-map-feature-id="{{ photo.map_feature_id }}" + data-label="{{ photo.label }}" data-map-feature-photo-detail-absolute-url="{{ photo.absolute_detail_url }}" data-map-feature-photo-image-absolute-url="{{ photo.absolute_image }}" data-map-feature-photo-thumbnail="{{ photo.thumbnail }}" @@ -18,9 +21,14 @@ {% trans 'Photo number' %} {{ forloop.counter }} {% if last_effective_instance_user|photo_is_deletable:photo.raw %} + + + {% endif %} + {% if photo.has_label %} + {% endif %}
{% empty %} diff --git a/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html b/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html index 84ef93bd2..a1494405f 100644 --- a/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html +++ b/opentreemap/treemap/templates/treemap/partials/photo_social_media_sharing.html @@ -7,25 +7,19 @@

{% trans "Photo Added!" %}

- diff --git a/opentreemap/treemap/urls.py b/opentreemap/treemap/urls.py index a148d4a41..56c3374fa 100644 --- a/opentreemap/treemap/urls.py +++ b/opentreemap/treemap/urls.py @@ -46,6 +46,8 @@ routes.map_feature_accordion, name='map_feature_accordion'), url('^features/(?P\d+)/photo/(?P\d+)/detail$', routes.map_feature_photo_detail, name='map_feature_photo_detail'), + url('^features/(?P\d+)/photo/(?P\d+)/label$', + routes.add_map_feature_photo_label, name='map_feature_photo_label'), url('^features/(?P\d+)/photo/(?P\d+)$', routes.map_feature_photo, name='map_feature_photo'), url(r'^features/(?P\d+)/favorite$', diff --git a/opentreemap/treemap/views/map_feature.py b/opentreemap/treemap/views/map_feature.py index b3cd7f007..869c7504e 100644 --- a/opentreemap/treemap/views/map_feature.py +++ b/opentreemap/treemap/views/map_feature.py @@ -22,7 +22,8 @@ from treemap.units import Convertible from treemap.models import (Tree, Species, MapFeature, - MapFeaturePhoto, TreePhoto, Favorite) + MapFeaturePhoto, TreePhoto, Favorite, + MapFeaturePhotoLabel) from treemap.util import (package_field_errors, to_object_name) from treemap.images import get_image_from_request @@ -398,6 +399,19 @@ def rotate_map_feature_photo(request, instance, feature_id, photo_id): mf_photo.save_with_user(request.user) +@get_photo_context_and_errors +def add_map_feature_photo_label(request, instance, feature_id, photo_id): + feature = get_map_feature_or_404(feature_id, instance) + photo_class = TreePhoto if feature.is_plot else MapFeaturePhoto + mf_photo = get_object_or_404(photo_class, pk=photo_id, map_feature=feature) + label_dict = json.loads(request.body) + map_feature_photo_label = MapFeaturePhotoLabel() + map_feature_photo_label.map_feature_photo = mf_photo + map_feature_photo_label.name = label_dict['label'] + map_feature_photo_label.save() + return + + @get_photo_context_and_errors def delete_photo(request, instance, feature_id, photo_id): feature = get_map_feature_or_404(feature_id, instance) From 955bb135ca163dfe5752a8ea1d36e19882fba1da Mon Sep 17 00:00:00 2001 From: Tom Z Date: Sat, 29 Feb 2020 15:36:29 -0500 Subject: [PATCH 03/71] issue-3287, a typo fix and a migration --- .../migrations/0047_auto_20200217_1043.py | 28 +++++++++++++++++++ opentreemap/treemap/models.py | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 opentreemap/treemap/migrations/0047_auto_20200217_1043.py diff --git a/opentreemap/treemap/migrations/0047_auto_20200217_1043.py b/opentreemap/treemap/migrations/0047_auto_20200217_1043.py new file mode 100644 index 000000000..e98f5207c --- /dev/null +++ b/opentreemap/treemap/migrations/0047_auto_20200217_1043.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2020-02-17 16:43 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('treemap', '0046_auto_20170907_0937'), + ] + + operations = [ + migrations.CreateModel( + name='MapFeaturePhotoLabel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=40)), + ('map_feature_photo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='treemap.MapFeaturePhoto')), + ], + ), + migrations.AlterUniqueTogether( + name='mapfeaturephotolabel', + unique_together=set([('map_feature_photo', 'name')]), + ), + ] diff --git a/opentreemap/treemap/models.py b/opentreemap/treemap/models.py index 78cb87c41..ce6a8c885 100644 --- a/opentreemap/treemap/models.py +++ b/opentreemap/treemap/models.py @@ -1510,7 +1510,7 @@ def __init__(self, *args, **kwargs): class MapFeaturePhotoLabel(models.Model): """ - Provide a tag for a phot + Provide a tag for a photo """ map_feature_photo = models.ForeignKey(MapFeaturePhoto) name = models.CharField(max_length=40) From 30873f59953fd18293a7c901c7b72e301b504bd0 Mon Sep 17 00:00:00 2001 From: Tom Z Date: Sat, 21 Mar 2020 15:22:02 -0400 Subject: [PATCH 04/71] develop - fix auth to use only strings --- opentreemap/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentreemap/api/auth.py b/opentreemap/api/auth.py index a4a0422a2..6fb488637 100644 --- a/opentreemap/api/auth.py +++ b/opentreemap/api/auth.py @@ -47,7 +47,7 @@ def get_signature_for_request(request, secret_key): sign_string += body_encoded sig = base64.b64encode( - hmac.new(secret_key, sign_string, hashlib.sha256).digest()) + hmac.new(str(secret_key), str(sign_string), hashlib.sha256).digest()) return sig From 2aa3f315a632ec68d1a6ad5f87cedcb1ec31766e Mon Sep 17 00:00:00 2001 From: Jason Biegel Date: Sun, 22 Mar 2020 00:42:41 -0400 Subject: [PATCH 05/71] created inat model, cant connect no username password --- __init__.py | 0 opentreemap/__init__.py | 0 .../opentreemap/integrations/__init__.py | 0 .../opentreemap/integrations/inaturalist.py | 118 +++++++++++ .../integrations/tests/__init__.py | 0 .../integrations/tests/test_inaturalist.py | 41 ++++ .../opentreemap/settings/default_settings.py | 1 + opentreemap/treemap/views/map_feature.py | 183 +----------------- requirements.txt | 1 + 9 files changed, 165 insertions(+), 179 deletions(-) create mode 100644 __init__.py create mode 100644 opentreemap/__init__.py create mode 100644 opentreemap/opentreemap/integrations/__init__.py create mode 100644 opentreemap/opentreemap/integrations/inaturalist.py create mode 100644 opentreemap/opentreemap/integrations/tests/__init__.py create mode 100644 opentreemap/opentreemap/integrations/tests/test_inaturalist.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/opentreemap/__init__.py b/opentreemap/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/opentreemap/opentreemap/integrations/__init__.py b/opentreemap/opentreemap/integrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/opentreemap/opentreemap/integrations/inaturalist.py b/opentreemap/opentreemap/integrations/inaturalist.py new file mode 100644 index 000000000..1df2e40f1 --- /dev/null +++ b/opentreemap/opentreemap/integrations/inaturalist.py @@ -0,0 +1,118 @@ +from datetime import timedelta + +import requests +from django.conf import settings +from django.db import connection +from background_task import background + +from treemap.models import INaturalistObservation + +base_url = "https://www.inaturalist.org" + + +def get_inaturalist_auth_token(): + + payload = { + 'client_id': settings.INATURALIST_APP_ID, + 'client_secret': settings.INATURALIST_APP_SECRET, + 'grant_type': 'password', + 'username': settings.USERNAME, + 'password': settings.PASSWORD + } + + r = requests.post( + url="{base_url}/oauth/token".format(base_url=base_url), + data=payload + ) + token = r.json()['access_token'] + return token + + +def create_observation(token, latitude, longitude): + + headers = {'Authorization': 'Bearer {}'.format(token)} + params = {'observation': { + 'observed_on_string': datetime.datetime.now().isoformat(), + 'latitude': latitude, + 'longitude': longitude + } + } + + response = requests.post( + url="{base_url}/observations.json".format(base_url=base_url), + json=params, + headers=headers + ) + + return response.json()[0] + + +def add_photo_to_observation(token, observation_id, photo): + + headers = {'Authorization': 'Bearer {}'.format(token)} + data = {'observation_photo[observation_id]': observation_id} + file_data = {'file': photo.image.file.file} + + requests.post( + url="{base_url}/observation_photos".format(base_url=base_url), + headers=headers, + data=data, + files=file_data + ) + + +@background(schedule=timedelta(hours=24)) +def sync_identifications_routine(): + """ + This helper function exists to make testing of the routine possible. + """ + sync_identifications() + +def get_o9n(o9n_id): + token = get_inaturalist_auth_token() + headers = {'Authorization': 'Bearer {}'.format(token)} + + response = requests.get( + url="{base_url}/observations/{o9n_id}".format( + base_url=base_url, o9n_id=o9n_id), + headers=headers + ) + import pdb; pdb.set_trace() + + +def sync_identifications(): + o9n_attr = INaturalistObservation.observation_id.field_name + + o9n_ids = INaturalistObservation.objects.filter( + is_identified=False).values(o9n_attr) + + for o9n_id in o9n_ids: + get_o9n(o9n_id[o9n_attr]) + + + +def get_features_for_inaturalist(): + """ + Get all the features that have a label and can be submitted to iNaturalist + """ + query = """ + SELECT photo.id, photo.map_feature_id, photo.instance_id + FROM treemap_mapfeaturephoto photo + JOIN treemap_mapfeaturephotolabel label on label.map_feature_photo_id = photo.id + LEFT JOIN treemap_inaturalistobservation inat on inat.map_feature_id = photo.map_feature_id + where 1=1 + and inat.id is null + group by photo.id, photo.map_feature_id, photo.instance_id + having sum(case when label.name = 'shape' then 1 else 0 end) > 0 + and sum(case when label.name = 'bark' then 1 else 0 end) > 0 + and sum(case when label.name = 'leaf' then 1 else 0 end) > 0 + """ + + with connection.cursor() as cursor: + cursor.execute(query) + results = cursor.fetchall() + + return [{'photo_id': r[0], + 'feature_id': r[1], + 'instance_id': r[2]} + for r in results] diff --git a/opentreemap/opentreemap/integrations/tests/__init__.py b/opentreemap/opentreemap/integrations/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/opentreemap/opentreemap/integrations/tests/test_inaturalist.py b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py new file mode 100644 index 000000000..9c0b76957 --- /dev/null +++ b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py @@ -0,0 +1,41 @@ +from django.contrib.gis.geos import Point + +from opentreemap.integrations import inaturalist +from treemap.models import MapFeature, INaturalistObservation, Tree, Plot +from treemap.tests.base import OTMTestCase +from treemap.tests import (make_instance, make_commander_user) + + +class TestINaturalist(OTMTestCase): + + def setUp(self): + self.instance = make_instance() + self.commander_user = make_commander_user(self.instance) + + def _createObservation(self, is_identified=False): + plot = Plot(geom=Point(0, 0), instance=self.instance) + plot.save_with_user(self.commander_user) + + tree = Tree(instance=self.instance, plot=plot) + tree.save_with_user(self.commander_user) + + o = INaturalistObservation(is_identified=is_identified, + observation_id=1, + map_feature=plot, + tree=tree) + o.save() + return o + + def test_no_observations(self): + inaturalist.sync_identifications() + + def test_identified(self): + self._createObservation(is_identified=True) + inaturalist.sync_identifications() + + def test_unidentified(self): + self._createObservation() + inaturalist.sync_identifications() + + + diff --git a/opentreemap/opentreemap/settings/default_settings.py b/opentreemap/opentreemap/settings/default_settings.py index 400572b63..600a91ef6 100644 --- a/opentreemap/opentreemap/settings/default_settings.py +++ b/opentreemap/opentreemap/settings/default_settings.py @@ -294,6 +294,7 @@ 'django.contrib.postgres', 'django_js_reverse', 'webpack_loader', + 'background_task' ) I18N_APPS = ( diff --git a/opentreemap/treemap/views/map_feature.py b/opentreemap/treemap/views/map_feature.py index 350c58485..829ab7527 100644 --- a/opentreemap/treemap/views/map_feature.py +++ b/opentreemap/treemap/views/map_feature.py @@ -7,7 +7,6 @@ import json import hashlib import re -import requests from functools import wraps from django.http import HttpResponse, HttpResponseRedirect @@ -20,6 +19,7 @@ from django.utils.translation import ugettext as _ from opentreemap.util import dotted_split +from opentreemap.integrations import inaturalist from treemap.lib.hide_at_zoom import (update_hide_at_zoom_after_move, update_hide_at_zoom_after_delete) @@ -475,135 +475,6 @@ def get_photo_id_from_photo_detail_url(url, feature_id): return int(re.match(r'.*/{}/photo/(\d+)/detail'.format(feature_id), url).groups()[0]) -def create_observation(token, latitude, longitude): - """ - """ - base_url = "https://www.inaturalist.org" - headers = {'Authorization': 'Bearer {}'.format(token)} - params = {'observation': { - 'observed_on_string': datetime.datetime.now().isoformat(), - 'latitude': latitude, - 'longitude': longitude - } - } - - response = requests.post( - url="{base_url}/observations.json".format(base_url=base_url), - json=params, - headers=headers - ) - ''' - return { - u'cached_votes_total': 0, - u'captive': False, - u'comments_count': 0, - u'community_taxon_id': None, - u'created_at': u'2020-01-07T22:04:44.757-05:00', - u'created_at_utc': u'2020-01-08T03:04:44.757Z', - u'delta': False, - u'description': None, - u'faves_count': 0, - u'geoprivacy': None, - u'iconic_taxon_id': None, - u'iconic_taxon_name': None, - u'id': 37388076, - u'id_please': False, - u'identifications_count': 0, - u'last_indexed_at': u'2020-01-07T19:04:49.322-08:00', - u'latitude': u'40.7083055556', - u'license': u'CC-BY-NC', - u'location_is_exact': False, - u'longitude': u'-74.0893888889', - u'map_scale': None, - u'mappable': True, - u'num_identification_agreements': 0, - u'num_identification_disagreements': 0, - u'oauth_application_id': 385, - u'observation_photos_count': 0, - u'observation_sounds_count': 0, - u'observed_on': u'2020-01-07', - u'observed_on_string': u'2020-01-07T21:04:44.393925', - u'old_uuid': None, - u'out_of_range': None, - u'owners_identification_from_vision': None, - u'place_guess': u'Hudson County, US-NJ, US', - u'positional_accuracy': None, - u'positioning_device': None, - u'positioning_method': None, - u'private_latitude': None, - u'private_longitude': None, - u'private_place_guess': None, - u'private_positional_accuracy': None, - u'project_observations': [], - u'public_positional_accuracy': None, - u'quality_grade': u'casual', - u'site_id': 1, - u'species_guess': None, - u'taxon_geoprivacy': None, - u'taxon_id': None, - u'time_observed_at': u'2020-01-07T21:04:44.000-05:00', - u'time_observed_at_utc': u'2020-01-08T02:04:44.000Z', - u'time_zone': u'America/New_York', - u'timeframe': None, - u'updated_at': u'2020-01-07T22:04:44.757-05:00', - u'updated_at_utc': u'2020-01-08T03:04:44.757Z', - u'uri': None, - u'user_id': 2384052, - u'user_login': u'tzinckgraf', - u'uuid': u'0e26bafc-bd23-48d0-9bda-806450093c88', - u'zic_time_zone': None - } - ''' - return response.json()[0] - - -def add_photo_to_observation(token, observation_id, photo): - base_url = "https://www.inaturalist.org" - headers = {'Authorization': 'Bearer {}'.format(token)} - data = {'observation_photo[observation_id]': observation_id} - file_data = {'file': photo.image.file.file} - - response = requests.post( - url="{base_url}/observation_photos".format(base_url=base_url), - headers=headers, - data=data, - files=file_data - ) - ''' - return { - u'created_at': u'2020-01-07T22:17:04.531-05:00', - u'created_at_utc': u'2020-01-08T03:17:04.531Z', - u'id': 54883602, - u'observation_id': 37388076, - u'old_uuid': None, - u'photo': {u'attribution': u'(c) Thomas Zinckgraf, some rights reserved (CC BY-NC)', - u'created_at': u'2020-01-07T22:17:02.709-05:00', - u'id': 59263063, - u'large_url': None, - u'license': 2, - u'license_name': u'Creative Commons Attribution-NonCommercial License', - u'license_url': u'http://creativecommons.org/licenses/by-nc/4.0/', - u'medium_url': None, - u'native_original_image_url': None, - u'native_page_url': None, - u'native_photo_id': u'59263063', - u'native_realname': u'Thomas Zinckgraf', - u'native_username': u'tzinckgraf', - u'small_url': None, - u'square_url': None, - u'subtype': None, - u'thumb_url': None, - u'type': u'LocalPhoto', - u'updated_at': u'2020-01-07T22:17:02.709-05:00', - u'user_id': 2384052}, - u'photo_id': 59263063, - u'position': None, - u'updated_at': u'2020-01-07T22:17:04.531-05:00', - u'updated_at_utc': u'2020-01-08T03:17:04.531Z', - u'uuid': u'db6b5a49-eb92-4b12-8d3f-83f388ac55f0'} - ''' - - def inaturalist_add(request, instance, *args, **kwargs): try: token = request.session['inaturalist_token'] @@ -622,65 +493,19 @@ def inaturalist_add(request, instance, *args, **kwargs): (longitude, latitude) = feature.latlon.coords - observation = create_observation(token, latitude, longitude) - photo_info = add_photo_to_observation(token, observation['id'], photo) + observation = inaturalist.create_observation(token, latitude, longitude) + photo_info = inaturalist.add_photo_to_observation(token, observation['id'], photo) return {'success': True} -def get_features_for_inaturalist(): - """ - Get all the features that have a label and can be submitted to iNaturalist - """ - query = """ - SELECT photo.id, photo.map_feature_id, photo.instance_id - FROM treemap_mapfeaturephoto photo - JOIN treemap_mapfeaturephotolabel label on label.map_feature_photo_id = photo.id - LEFT JOIN treemap_inaturalistobservation inat on inat.map_feature_id = photo.map_feature_id - where 1=1 - and inat.id is null - group by photo.id, photo.map_feature_id, photo.instance_id - having sum(case when label.name = 'shape' then 1 else 0 end) > 0 - and sum(case when label.name = 'bark' then 1 else 0 end) > 0 - and sum(case when label.name = 'leaf' then 1 else 0 end) > 0 - """ - - with connection.cursor() as cursor: - cursor.execute(query) - results = cursor.fetchall() - - return [{'photo_id': r[0], - 'feature_id': r[1], - 'instance_id': r[2]} - for r in results] - - -def get_inaturalist_auth_token(): - base_url = "https://www.inaturalist.org" - - payload = { - 'client_id': settings.INATURALIST_APP_ID, - 'client_secret': settings.INATURALIST_APP_SECRET, - 'grant_type': 'password', - 'username': settings.USERNAME, - 'password': settings.PASSWORD - } - - r = requests.post( - url="{base_url}/oauth/token".format(base_url=base_url), - data=payload - ) - token = r.json()['access_token'] - return token - - def inaturalist_create_observations(request, instance, *args, **kwargs): features = get_features_for_inaturalist() if not features: return - token = get_inaturalist_auth_token() + token = inaturalist.get_inaturalist_auth_token() for feature in features: feature = get_map_feature_or_404(feature['feature_id'], instance) diff --git a/requirements.txt b/requirements.txt index 8affc6203..b33ae946a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ chardet==3.0.4 # https://docs.djangoproject.com/en/1.10/releases/#id2 Django==1.11.16 # rq.filter: >=1.11,<1.12 django-apptemplates==1.3 +django-background-tasks==1.2.5 django-contrib-comments==1.8.0 django-js-reverse==0.7.3 django-queryset-csv==1.0.2 # https://github.com/azavea/django-queryset-csv/commits/master From bfa214a6ec362c9db31d21facff8cc0112361410 Mon Sep 17 00:00:00 2001 From: Jason Biegel Date: Sun, 22 Mar 2020 01:37:58 -0400 Subject: [PATCH 06/71] still need to add logging and test fixture/network mocks --- .../opentreemap/integrations/inaturalist.py | 34 ++++++++++--------- .../integrations/tests/test_inaturalist.py | 4 ++- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/opentreemap/opentreemap/integrations/inaturalist.py b/opentreemap/opentreemap/integrations/inaturalist.py index 1df2e40f1..d25b3b496 100644 --- a/opentreemap/opentreemap/integrations/inaturalist.py +++ b/opentreemap/opentreemap/integrations/inaturalist.py @@ -1,3 +1,4 @@ +import dateutil.parser from datetime import timedelta import requests @@ -5,7 +6,7 @@ from django.db import connection from background_task import background -from treemap.models import INaturalistObservation +from treemap.models import INaturalistObservation, Species base_url = "https://www.inaturalist.org" @@ -68,27 +69,28 @@ def sync_identifications_routine(): """ sync_identifications() -def get_o9n(o9n_id): - token = get_inaturalist_auth_token() - headers = {'Authorization': 'Bearer {}'.format(token)} - response = requests.get( - url="{base_url}/observations/{o9n_id}".format( - base_url=base_url, o9n_id=o9n_id), - headers=headers - ) - import pdb; pdb.set_trace() +def get_o9n(o9n_id): + return requests.get( + url="{base_url}/observations/{o9n_id}.json".format( + base_url=base_url, o9n_id=o9n_id) + ).json() -def sync_identifications(): - o9n_attr = INaturalistObservation.observation_id.field_name +def _set_identification(o9n_model, taxon): + o9n_model.tree.species = Species(common_name=taxon['common_name']['name']) + o9n_model.identified_at = dateutil.parser.parse(taxon['updated_at']) + o9n_model.is_identified = True + o9n_model.save() - o9n_ids = INaturalistObservation.objects.filter( - is_identified=False).values(o9n_attr) - for o9n_id in o9n_ids: - get_o9n(o9n_id[o9n_attr]) +def sync_identifications(): + o9n_models = INaturalistObservation.objects.filter(is_identified=False) + for o9n_model in o9n_models: + o9n_json = get_o9n(o9n_model.observation_id) + if 'taxon' in o9n_json: + _set_identification(o9n_model, o9n_json['taxon']) def get_features_for_inaturalist(): diff --git a/opentreemap/opentreemap/integrations/tests/test_inaturalist.py b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py index 9c0b76957..ab31dace8 100644 --- a/opentreemap/opentreemap/integrations/tests/test_inaturalist.py +++ b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py @@ -20,7 +20,7 @@ def _createObservation(self, is_identified=False): tree.save_with_user(self.commander_user) o = INaturalistObservation(is_identified=is_identified, - observation_id=1, + observation_id=39496756, map_feature=plot, tree=tree) o.save() @@ -35,7 +35,9 @@ def test_identified(self): def test_unidentified(self): self._createObservation() + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 1) inaturalist.sync_identifications() + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) From b5d3e5aedc26f75f7ff2d77a2a71204ec5fccf17 Mon Sep 17 00:00:00 2001 From: Jason Biegel Date: Sun, 22 Mar 2020 13:38:23 -0400 Subject: [PATCH 07/71] fixture and network mock, still need to add logging --- .../integrations/tests/fixtures/__init__.py | 15 + .../tests/fixtures/observation.json | 301 ++++++++++++++++++ .../integrations/tests/test_inaturalist.py | 42 ++- test-requirements.txt | 4 +- 4 files changed, 352 insertions(+), 10 deletions(-) create mode 100644 opentreemap/opentreemap/integrations/tests/fixtures/__init__.py create mode 100644 opentreemap/opentreemap/integrations/tests/fixtures/observation.json diff --git a/opentreemap/opentreemap/integrations/tests/fixtures/__init__.py b/opentreemap/opentreemap/integrations/tests/fixtures/__init__.py new file mode 100644 index 000000000..3839ec09c --- /dev/null +++ b/opentreemap/opentreemap/integrations/tests/fixtures/__init__.py @@ -0,0 +1,15 @@ +import copy +import json + +# API reference: https://www.inaturalist.org/pages/api+reference#get-observations-id +with open('opentreemap/integrations/tests/fixtures/observation.json') as json_file: + _o9n = json.loads(json_file.read()) + + +def get_inaturalist_o9n(o9n_id=None): + o9n_copy = copy.deepcopy(_o9n) + + if o9n_id: + o9n_copy['id'] = o9n_id + + return o9n_copy diff --git a/opentreemap/opentreemap/integrations/tests/fixtures/observation.json b/opentreemap/opentreemap/integrations/tests/fixtures/observation.json new file mode 100644 index 000000000..c579c7dfa --- /dev/null +++ b/opentreemap/opentreemap/integrations/tests/fixtures/observation.json @@ -0,0 +1,301 @@ +{ + "id": 32189837, + "observed_on": "2019-09-05", + "description": null, + "latitude": "42.355992389", + "longitude": "-74.1652624123", + "map_scale": null, + "timeframe": null, + "species_guess": null, + "user_id": 0, + "taxon_id": 48678, + "created_at": "2019-09-05T23:14:55.466Z", + "updated_at": "2019-09-05T23:17:25.894Z", + "place_guess": "Nowhere, NY, US", + "id_please": false, + "observed_on_string": "Thu Sep 05 2019 19:12:11 GMT-0400 (EDT)", + "iconic_taxon_id": 47126, + "num_identification_agreements": 0, + "num_identification_disagreements": 0, + "time_observed_at": "2019-09-05T23:12:11.000Z", + "time_zone": "Eastern Time (US & Canada)", + "location_is_exact": false, + "delta": false, + "positional_accuracy": 30, + "private_latitude": null, + "private_longitude": null, + "private_positional_accuracy": null, + "geoprivacy": null, + "quality_grade": "needs_id", + "positioning_method": null, + "positioning_device": null, + "out_of_range": null, + "license": null, + "uri": "https://www.inaturalist.org/observations/1", + "observation_photos_count": 1, + "comments_count": 0, + "zic_time_zone": "America/New_York", + "oauth_application_id": 3, + "observation_sounds_count": 0, + "identifications_count": 1, + "captive": false, + "community_taxon_id": null, + "site_id": 1, + "old_uuid": null, + "public_positional_accuracy": 30, + "mappable": true, + "cached_votes_total": 0, + "last_indexed_at": "2020-03-22T12:38:47.845Z", + "private_place_guess": null, + "uuid": "b4722d45-eab0-476a-b0ed-e99ac521ec6e", + "taxon_geoprivacy": null, + "user_login": "anonymous", + "iconic_taxon_name": "Plantae", + "captive_flag": false, + "created_at_utc": "2019-09-05T23:14:55.466Z", + "updated_at_utc": "2019-09-05T23:17:25.894Z", + "time_observed_at_utc": "2019-09-05T23:12:11.000Z", + "faves_count": 0, + "owners_identification_from_vision": true, + "user": { + "id": 2132915, + "login": "anonymous", + "name": "Anon", + "observations_count": 44, + "identifications_count": 0, + "user_icon_url": "https://static.inaturalist.org/attachments/users/icons/0/thumb.jpeg?123456789", + "medium_user_icon_url": "https://static.inaturalist.org/attachments/users/icons/0/medium.jpeg?123456789", + "original_user_icon_url": "https://static.inaturalist.org/attachments/users/icons/0/original.jpeg?123456789" + }, + "observation_field_values": [], + "project_observations": [ + { + "id": 31875070, + "project_id": 4034, + "observation_id": 32189837, + "created_at": "2019-09-05T23:30:51.927Z", + "updated_at": "2019-09-05T23:30:51.927Z", + "curator_identification_id": null, + "tracking_code": null, + "user_id": null, + "uuid": "507a7703-b307-498d-bbac-d4dbbb49f06c", + "project": { + "id": 4034, + "title": "New York Wildflower Monitoring Project", + "description": "Much remains to be discovered about the flora of New York and why plants grow where and how they do. This project was initiated in 2015 to expand upon efforts to document plant observations during class field trips. The primary goal was to build an online photographic guide to common flowering plants across New York while exploring ways to connect citizen scientists, improve accuracy in plant monitoring, and identify limitations to implementing monitoring projects. \r\n\r\nIn September 2017, we will no longer be tracking this project regularly; however, any observations of flowering plants in New York will still be automatically added to the project. We appreciate your interest and enthusiasm for learning more about the different plant communities of New York and hope this continues to be a useful educational tool moving forward. \r\n", + "icon_url": "https://static.inaturalist.org/projects/4034-icon-span2.JPG?1505523585" + } + } + ], + "observation_photos": [ + { + "id": 46563675, + "observation_id": 32189837, + "photo_id": 50463281, + "position": 0, + "created_at": "2019-09-05T23:17:25.843Z", + "updated_at": "2019-09-05T23:17:25.843Z", + "old_uuid": null, + "uuid": "eabe8724-ea94-42a6-b2b9-b82a70e8f51c", + "photo": { + "id": 50463281, + "square_url": "https://static.inaturalist.org/photos/50463281/square.jpg?1567725442", + "thumb_url": "https://static.inaturalist.org/photos/50463281/thumb.jpg?1567725442", + "small_url": "https://static.inaturalist.org/photos/50463281/small.jpg?1567725442", + "medium_url": "https://static.inaturalist.org/photos/50463281/medium.jpg?1567725442", + "large_url": "https://static.inaturalist.org/photos/50463281/large.jpg?1567725442", + "created_at": "2019-09-05T23:17:24.222Z", + "updated_at": "2019-09-05T23:17:24.222Z", + "native_page_url": "https://www.inaturalist.org/photos/50463281", + "native_username": "anon", + "license": 0, + "subtype": null, + "native_original_image_url": null, + "uuid": "b1a4ab8b-f0ba-48f7-be5b-4d83d111c610", + "license_code": "C", + "attribution": "(c) Anon, all rights reserved", + "license_name": "Copyright", + "license_url": "http://en.wikipedia.org/wiki/Copyright", + "type": "LocalPhoto" + } + } + ], + "comments": [], + "taxon": { + "id": 48678, + "name": "Solidago", + "rank": "genus", + "source_id": 1, + "created_at": "2008-11-07T06:14:11.000Z", + "updated_at": "2019-12-02T22:14:42.762Z", + "iconic_taxon_id": 47126, + "is_iconic": false, + "name_provider": "ColNameProvider", + "observations_count": 74165, + "listed_taxa_count": 19714, + "rank_level": 20, + "unique_name": "gyldenris", + "wikipedia_summary": "Solidago, commonly called goldenrods, is a genus of about 100 to 120 species of flowering plants in the aster family, Asteraceae. Most are herbaceous perennial species found in open areas such as meadows, prairies, and savannas. They are mostly native to North America, including Mexico; a few species are native to South America and Eurasia. Some American species have also been introduced into Europe and other parts of the world.", + "wikipedia_title": "", + "ancestry": "48460/47126/211194/47125/47124/47605/47604/632790/461542/972606", + "conservation_status": null, + "conservation_status_source_id": null, + "conservation_status_source_identifier": null, + "is_active": true, + "complete": null, + "complete_rank": null, + "taxon_framework_relationship_id": 316691, + "uuid": "ca854959-47e8-4d7e-b4af-156fc9d00236", + "default_name": { + "id": 221777, + "name": "goldenrods", + "is_valid": true, + "lexicon": "English" + }, + "photo_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "iconic_taxon_name": "Plantae", + "conservation_status_name": null, + "image_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "common_name": { + "id": 221777, + "name": "goldenrods", + "is_valid": true, + "lexicon": "English" + }, + "taxon_photos": [ + { + "id": 15264, + "taxon_id": 48678, + "photo_id": 30546, + "position": null, + "created_at": null, + "updated_at": null, + "photo": { + "id": 30546, + "user_id": null, + "native_photo_id": "4032129702", + "square_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "thumb_url": "https://static.inaturalist.org/photos/30546/thumb.jpg?1545409047", + "small_url": "https://static.inaturalist.org/photos/30546/small.jpg?1545409047", + "medium_url": "https://static.inaturalist.org/photos/30546/medium.jpg?1545409047", + "large_url": "https://static.inaturalist.org/photos/30546/large.jpg?1545409047", + "created_at": "2011-05-02T04:50:39.000Z", + "updated_at": "2018-12-21T16:17:29.590Z", + "native_page_url": "http://www.flickr.com/photos/35478170@N08/4032129702", + "native_username": "Anon", + "native_realname": "Anon", + "license": 5, + "subtype": "FlickrPhoto", + "native_original_image_url": "https://farm3.staticflickr.com/2565/4032129702_b52cdb7ba3_o.jpg", + "uuid": "8a980b17-04b7-46c7-be44-5a02d322809a", + "license_code": "CC-BY-SA", + "attribution": "(c) Anon, some rights reserved (CC BY-SA)", + "license_name": "Creative Commons Attribution-ShareAlike License", + "license_url": "http://creativecommons.org/licenses/by-sa/4.0/", + "type": "LocalPhoto" + } + } + ] + }, + "identifications": [ + { + "id": 69767601, + "observation_id": 32189837, + "taxon_id": 48678, + "user_id": 2132915, + "body": null, + "created_at": "2019-09-05T23:14:55.509Z", + "updated_at": "2019-09-05T23:14:55.509Z", + "current": true, + "taxon_change_id": null, + "category": "leading", + "uuid": "9ad16252-4ac8-4d6b-9781-0ea99f26d07c", + "blind": null, + "previous_observation_taxon_id": 48678, + "disagreement": false, + "user": { + "id": 2132915, + "login": "anon", + "name": "Anon", + "user_icon_url": "https://static.inaturalist.org/attachments/users/icons/0/thumb.jpeg?123456789" + }, + "taxon": { + "id": 48678, + "name": "Solidago", + "rank": "genus", + "source_id": 1, + "created_at": "2008-11-07T06:14:11.000Z", + "updated_at": "2019-12-02T22:14:42.762Z", + "iconic_taxon_id": 47126, + "is_iconic": false, + "name_provider": "ColNameProvider", + "observations_count": 74165, + "listed_taxa_count": 19714, + "rank_level": 20, + "unique_name": "gyldenris", + "wikipedia_summary": "Solidago, commonly called goldenrods, is a genus of about 100 to 120 species of flowering plants in the aster family, Asteraceae. Most are herbaceous perennial species found in open areas such as meadows, prairies, and savannas. They are mostly native to North America, including Mexico; a few species are native to South America and Eurasia. Some American species have also been introduced into Europe and other parts of the world.", + "wikipedia_title": "", + "ancestry": "48460/47126/211194/47125/47124/47605/47604/632790/461542/972606", + "conservation_status": null, + "conservation_status_source_id": null, + "conservation_status_source_identifier": null, + "is_active": true, + "complete": null, + "complete_rank": null, + "taxon_framework_relationship_id": 316691, + "uuid": "ca854959-47e8-4d7e-b4af-156fc9d00236", + "default_name": { + "id": 221777, + "name": "goldenrods", + "is_valid": true, + "lexicon": "English" + }, + "photo_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "iconic_taxon_name": "Plantae", + "conservation_status_name": null, + "image_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "common_name": { + "id": 221777, + "name": "goldenrods", + "is_valid": true, + "lexicon": "English" + }, + "taxon_photos": [ + { + "id": 15264, + "taxon_id": 48678, + "photo_id": 30546, + "position": null, + "created_at": null, + "updated_at": null, + "photo": { + "id": 30546, + "user_id": null, + "native_photo_id": "4032129702", + "square_url": "https://static.inaturalist.org/photos/30546/square.jpg?1545409047", + "thumb_url": "https://static.inaturalist.org/photos/30546/thumb.jpg?1545409047", + "small_url": "https://static.inaturalist.org/photos/30546/small.jpg?1545409047", + "medium_url": "https://static.inaturalist.org/photos/30546/medium.jpg?1545409047", + "large_url": "https://static.inaturalist.org/photos/30546/large.jpg?1545409047", + "created_at": "2011-05-02T04:50:39.000Z", + "updated_at": "2018-12-21T16:17:29.590Z", + "native_page_url": "http://www.flickr.com/photos/35478170@N08/4032129702", + "native_username": "Anon", + "native_realname": "Anon", + "license": 5, + "subtype": "FlickrPhoto", + "native_original_image_url": "https://farm3.staticflickr.com/2565/4032129702_b52cdb7ba3_o.jpg", + "uuid": "8a980b17-04b7-46c7-be44-5a02d322809a", + "license_code": "CC-BY-SA", + "attribution": "(c) Anon, some rights reserved (CC BY-SA)", + "license_name": "Creative Commons Attribution-ShareAlike License", + "license_url": "http://creativecommons.org/licenses/by-sa/4.0/", + "type": "LocalPhoto" + } + } + ] + } + } + ], + "faves": [] +} diff --git a/opentreemap/opentreemap/integrations/tests/test_inaturalist.py b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py index ab31dace8..a50374a80 100644 --- a/opentreemap/opentreemap/integrations/tests/test_inaturalist.py +++ b/opentreemap/opentreemap/integrations/tests/test_inaturalist.py @@ -1,18 +1,24 @@ from django.contrib.gis.geos import Point +from mock import patch from opentreemap.integrations import inaturalist +from opentreemap.integrations.tests import fixtures from treemap.models import MapFeature, INaturalistObservation, Tree, Plot from treemap.tests.base import OTMTestCase from treemap.tests import (make_instance, make_commander_user) class TestINaturalist(OTMTestCase): + instance = None + commander_user = None + + GET_O9N_TARGET = 'opentreemap.integrations.inaturalist.get_o9n' def setUp(self): self.instance = make_instance() self.commander_user = make_commander_user(self.instance) - def _createObservation(self, is_identified=False): + def _create_observation(self, o9n_id=32189837, is_identified=False): plot = Plot(geom=Point(0, 0), instance=self.instance) plot.save_with_user(self.commander_user) @@ -20,24 +26,42 @@ def _createObservation(self, is_identified=False): tree.save_with_user(self.commander_user) o = INaturalistObservation(is_identified=is_identified, - observation_id=39496756, + observation_id=o9n_id, map_feature=plot, tree=tree) o.save() return o def test_no_observations(self): - inaturalist.sync_identifications() + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) + + with patch(TestINaturalist.GET_O9N_TARGET) as get_o9n_mock: + inaturalist.sync_identifications() + + get_o9n_mock.assert_not_called() + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) def test_identified(self): - self._createObservation(is_identified=True) - inaturalist.sync_identifications() + self._create_observation(is_identified=True) - def test_unidentified(self): - self._createObservation() - self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 1) - inaturalist.sync_identifications() self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) + with patch(TestINaturalist.GET_O9N_TARGET) as get_o9n_mock: + inaturalist.sync_identifications() + get_o9n_mock.assert_not_called() + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) + def test_unidentified(self): + o9n_id = 1 + + self._create_observation(o9n_id) + + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 1) + + with patch(TestINaturalist.GET_O9N_TARGET, + return_value=fixtures.get_inaturalist_o9n(o9n_id)) as get_o9n_mock: + inaturalist.sync_identifications() + + get_o9n_mock.assert_called_once_with(o9n_id) + self.assertEqual(INaturalistObservation.objects.filter(is_identified=False).count(), 0) diff --git a/test-requirements.txt b/test-requirements.txt index 679ced8a5..e00f587ee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ coverage==4.2 # https://coverage.readthedocs.io/en/coverage-4.2/changes.html PyVirtualDisplay==0.2 # https://github.com/ponty/PyVirtualDisplay/releases -# We use Firefox and selenium for running UI tests. But using the latest +# We use Firefox and selenium for running UI tests. But using the latest # versions is fragile because every few months a Firefox release is incompatible # with the current selenium. Since our UI tests don't depend on the latest features # of either we now use specific versions known to work together. @@ -11,3 +11,5 @@ PyVirtualDisplay==0.2 # https://github.com/ponty/PyVirtualDisplay/release # https://github.com/SeleniumHQ/selenium/blob/master/py/CHANGES selenium==2.53.6 # rq.filter: ==2.53.6 + +mock==3.0.5 From 95d9dd58eec52632ac155337d149ba859c996a33 Mon Sep 17 00:00:00 2001 From: Jason Biegel Date: Sun, 22 Mar 2020 21:11:42 -0400 Subject: [PATCH 08/71] docs and remove background_tasks --- .../opentreemap/integrations/inaturalist.py | 28 +++++++++---------- .../opentreemap/settings/default_settings.py | 1 - 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/opentreemap/opentreemap/integrations/inaturalist.py b/opentreemap/opentreemap/integrations/inaturalist.py index d25b3b496..4fae7d3a7 100644 --- a/opentreemap/opentreemap/integrations/inaturalist.py +++ b/opentreemap/opentreemap/integrations/inaturalist.py @@ -4,7 +4,6 @@ import requests from django.conf import settings from django.db import connection -from background_task import background from treemap.models import INaturalistObservation, Species @@ -62,15 +61,25 @@ def add_photo_to_observation(token, observation_id, photo): ) -@background(schedule=timedelta(hours=24)) -def sync_identifications_routine(): +def sync_identifications(): """ - This helper function exists to make testing of the routine possible. + Goes through all unidentified observations and updates them with taxonomy on iNaturalist """ - sync_identifications() + o9n_models = INaturalistObservation.objects.filter(is_identified=False) + + for o9n_model in o9n_models: + taxonomy = get_o9n(o9n_model.observation_id).get('taxon') + if taxonomy: + _set_identification(o9n_model, taxonomy) def get_o9n(o9n_id): + """ + Retrieve iNaturalist observation by ID + API docs: https://www.inaturalist.org/pages/api+reference#get-observations-id + :param o9n_id: observation ID + :return: observation JSON as a dict + """ return requests.get( url="{base_url}/observations/{o9n_id}.json".format( base_url=base_url, o9n_id=o9n_id) @@ -84,15 +93,6 @@ def _set_identification(o9n_model, taxon): o9n_model.save() -def sync_identifications(): - o9n_models = INaturalistObservation.objects.filter(is_identified=False) - - for o9n_model in o9n_models: - o9n_json = get_o9n(o9n_model.observation_id) - if 'taxon' in o9n_json: - _set_identification(o9n_model, o9n_json['taxon']) - - def get_features_for_inaturalist(): """ Get all the features that have a label and can be submitted to iNaturalist diff --git a/opentreemap/opentreemap/settings/default_settings.py b/opentreemap/opentreemap/settings/default_settings.py index 600a91ef6..400572b63 100644 --- a/opentreemap/opentreemap/settings/default_settings.py +++ b/opentreemap/opentreemap/settings/default_settings.py @@ -294,7 +294,6 @@ 'django.contrib.postgres', 'django_js_reverse', 'webpack_loader', - 'background_task' ) I18N_APPS = ( From 362a955e037d87b725e42082ac0451de8a232509 Mon Sep 17 00:00:00 2001 From: Jason Biegel Date: Sun, 22 Mar 2020 21:19:32 -0400 Subject: [PATCH 09/71] remove background task --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b33ae946a..8affc6203 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ chardet==3.0.4 # https://docs.djangoproject.com/en/1.10/releases/#id2 Django==1.11.16 # rq.filter: >=1.11,<1.12 django-apptemplates==1.3 -django-background-tasks==1.2.5 django-contrib-comments==1.8.0 django-js-reverse==0.7.3 django-queryset-csv==1.0.2 # https://github.com/azavea/django-queryset-csv/commits/master From 8cf512057f65dd130f0bc8881894e4aa0f45ba0e Mon Sep 17 00:00:00 2001 From: Tom Z Date: Mon, 30 Mar 2020 08:20:30 -0400 Subject: [PATCH 10/71] custom-fields-web, moved the custom fields on the website as well --- .../templates/treemap/map-add-tree.html | 29 ++++++++-- opentreemap/treemap/views/map_feature.py | 41 +------------- opentreemap/treemap/views/misc.py | 54 +++++++++++++++++-- 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/opentreemap/treemap/templates/treemap/map-add-tree.html b/opentreemap/treemap/templates/treemap/map-add-tree.html index 5bf1726ed..7ce687574 100644 --- a/opentreemap/treemap/templates/treemap/map-add-tree.html +++ b/opentreemap/treemap/templates/treemap/map-add-tree.html @@ -19,9 +19,32 @@

{% trans "Add a Tree" %}

{% trans "Trunk Diameter" as diameter %} {% create diameter from "Tree.diameter" in request.instance withtemplate "treemap/field/diameter_div.html" %} - {% for label, identifier in fields_for_add_tree %} - {% create label from identifier in request.instance withtemplate "treemap/field/div.html" %} - {% endfor %} + {% for group in field_groups %} + {% if group.model == "plot" %} +

{% trans "Plot Information" %}

+ + + {% for tuple in group.fields %} + {% with field=tuple.0 label=tuple.1 template=tuple.2 %} + {% create label from field in request.instance withtemplate template %} + {% endwith %} + {% endfor %} + +
+ + {% elif group.model == "tree" %} +

{% trans "Tree Information" %}

+ + + {% for tuple in group.fields %} + {% with field=tuple.0 label=tuple.1 template=tuple.2 %} + {% create label from field in request.instance withtemplate template %} + {% endwith %} + {% endfor %} + +
+ {% endif %} + {% endfor %} {% include 'treemap/partials/hidden_address.html' with object_name='plot' %} diff --git a/opentreemap/treemap/views/map_feature.py b/opentreemap/treemap/views/map_feature.py index 350c58485..91e77629d 100644 --- a/opentreemap/treemap/views/map_feature.py +++ b/opentreemap/treemap/views/map_feature.py @@ -31,12 +31,11 @@ from treemap.images import get_image_from_request from treemap.lib.photo import context_dict_for_photo -from treemap.lib.object_caches import udf_defs from treemap.lib.map_feature import (get_map_feature_or_404, raise_non_instance_404, context_dict_for_plot, context_dict_for_resource) -from treemap.views.misc import add_map_info_to_context +from treemap.views.misc import add_map_info_to_context, add_plot_field_groups def _request_to_update_map_feature(request, feature): @@ -108,7 +107,7 @@ def _map_feature_detail_context(request, instance, feature_id, edit=False): if feature.is_plot: partial = 'treemap/partials/plot_detail.html' - _add_plot_field_groups(context, instance) + add_plot_field_groups(context, instance) else: app = feature.__module__.split('.')[0] partial = '%s/%s_detail.html' % (app, feature.feature_type) @@ -116,42 +115,6 @@ def _map_feature_detail_context(request, instance, feature_id, edit=False): return context, partial -def _add_plot_field_groups(context, instance): - templates = { - "tree.id": "treemap/field/tree_id_tr.html", - "tree.species": "treemap/field/species_tr.html", - "tree.diameter": "treemap/field/diameter_tr.html" - } - - labels = { - # 'plot-species' is used as the "label" in the 'field' tag, - # but ulitmately gets used as an identifier in the template - "tree.species": "plot-species", - "tree.diameter": _("Trunk Diameter") - } - labels.update({ - v: k for k, v in context['tree'].scalar_udf_names_and_fields}) - labels.update({ - v: k for k, v in context['plot'].scalar_udf_names_and_fields}) - - def info(group): - group['fields'] = [ - (field, labels.get(field), - templates.get(field, "treemap/field/tr.html")) - for field in group.get('field_keys', []) - ] - group['collection_udfs'] = [ - next(udf for udf in udf_defs(instance) - if udf.full_name == udf_name) - for udf_name in group.get('collection_udf_keys', []) - ] - - return group - - context['field_groups'] = [ - info(group) for group in instance.web_detail_fields] - - def render_map_feature_detail_partial(request, instance, feature_id, **kwargs): context, partial = _map_feature_detail_context( request, instance, feature_id) diff --git a/opentreemap/treemap/views/misc.py b/opentreemap/treemap/views/misc.py index 608bac6f7..cbf8b9809 100644 --- a/opentreemap/treemap/views/misc.py +++ b/opentreemap/treemap/views/misc.py @@ -18,13 +18,14 @@ from stormwater.models import PolygonalMapFeature -from treemap.models import User, Species, StaticPage, Instance, Boundary +from treemap.models import User, Species, StaticPage, Instance, Boundary, Tree, Plot from treemap.plugin import get_viewable_instances_filter from treemap.lib.user import get_audits, get_audits_params from treemap.lib import COLOR_RE from treemap.lib.perms import model_is_creatable +from treemap.lib.object_caches import udf_defs from treemap.units import get_unit_abbreviation, get_units from treemap.util import leaf_models_of_class @@ -79,15 +80,17 @@ def get_map_view_context(request, instance): resource_classes = [] context = { - 'fields_for_add_tree': [ - (_('Tree Height'), 'Tree.height') - ], 'resource_classes': resource_classes, 'only_one_resource_class': len(resource_classes) == 1, 'polygon_area_units': get_unit_abbreviation( get_units(instance, 'greenInfrastructure', 'area')), 'q': request.GET.get('q'), } + add_plot_field_groups( + context, + instance, + filter_fields=['tree.id', 'tree.species', 'tree.diameter'] + ) add_map_info_to_context(context, instance) return context @@ -269,3 +272,46 @@ def inner_fn(request): return response return inner_fn + + +def add_plot_field_groups(context, instance, filter_fields=None): + _filter_fields = filter_fields or [] + templates = { + "tree.id": "treemap/field/tree_id_tr.html", + "tree.species": "treemap/field/species_tr.html", + "tree.diameter": "treemap/field/diameter_tr.html" + } + + labels = { + # 'plot-species' is used as the "label" in the 'field' tag, + # but ulitmately gets used as an identifier in the template + "tree.species": "plot-species", + "tree.diameter": _("Trunk Diameter") + } + + # use the tree if it exists to get the UDF fields, otherwise use a blank tree + _tree = context.get('tree', Tree()) + labels.update({ + v: k for k, v in _tree.scalar_udf_names_and_fields}) + + # use the plot if it exists to get the UDF fields, otherwise use a blank plot + _plot = context.get('plot', Plot()) + labels.update({ + v: k for k, v in _plot.scalar_udf_names_and_fields}) + + def info(group): + group['fields'] = [ + (field, labels.get(field), + templates.get(field, "treemap/field/tr.html")) + for field in group.get('field_keys', []) if field not in _filter_fields + ] + group['collection_udfs'] = [ + next(udf for udf in udf_defs(instance) + if udf.full_name == udf_name) + for udf_name in group.get('collection_udf_keys', []) + ] + + return group + + context['field_groups'] = [ + info(group) for group in instance.web_detail_fields] From ff1db30bd85c580e2346d3823668e563103f27fd Mon Sep 17 00:00:00 2001 From: Tom Z Date: Sun, 5 Apr 2020 17:25:21 -0400 Subject: [PATCH 11/71] ui-fixes-for-demo-20200405 - various UI fixes, which include - Minor renames - Additional step as placeholder for adding photos of different labels - Typeahead for species is now a dropdown as well - Species is required, unless the missing site is explicitly set --- .../treemap/js/src/mapPage/addTreeMode.js | 12 +++++-- opentreemap/treemap/js/src/mapPage/modes.js | 2 +- .../templates/treemap/field/species_div.html | 4 +++ .../templates/treemap/map-add-tree.html | 32 +++++++++++++++++-- .../treemap/partials/plot_detail.html | 2 +- opentreemap/treemap/views/map_feature.py | 18 +++++++++++ 6 files changed, 63 insertions(+), 7 deletions(-) diff --git a/opentreemap/treemap/js/src/mapPage/addTreeMode.js b/opentreemap/treemap/js/src/mapPage/addTreeMode.js index 01061af8c..dc801867a 100644 --- a/opentreemap/treemap/js/src/mapPage/addTreeMode.js +++ b/opentreemap/treemap/js/src/mapPage/addTreeMode.js @@ -11,13 +11,15 @@ var $ = require('jquery'), var activateMode = _.identity, deactivateMode = _.identity, STEP_LOCATE = 0, - STEP_DETAILS = 1, - STEP_FINAL = 2; + STEP_PHOTO = 1, + STEP_DETAILS = 2, + STEP_FINAL = 3; function init(options) { var $sidebar = $(options.sidebar), $speciesTypeahead = U.$find('#add-tree-species-typeahead', $sidebar), $summaryHead = U.$find('.summaryHead', $sidebar), + $isEmptySite = U.$find('#is-empty-site', $sidebar), $summarySubhead = U.$find('.summarySubhead', $sidebar), typeahead = otmTypeahead.create(options.typeahead), clearEditControls = function() { @@ -55,9 +57,13 @@ function init(options) { function aTreeFieldIsSet() { var data = manager.getFormData(); - return _.some(data, function (value, key) { + var treeValueSet = _.some(data, function (value, key) { return key && key.indexOf('tree') === 0 && value; }); + + // either we have a tree value set, or we have not explicitly + // said this is an empty site + return treeValueSet || !$isEmptySite.is(":checked"); } // In case we're adding another tree, make user move the marker diff --git a/opentreemap/treemap/js/src/mapPage/modes.js b/opentreemap/treemap/js/src/mapPage/modes.js index fced17767..1f18823dd 100644 --- a/opentreemap/treemap/js/src/mapPage/modes.js +++ b/opentreemap/treemap/js/src/mapPage/modes.js @@ -209,7 +209,7 @@ function getSpeciesTypeaheadOptions(idPrefix) { hidden: "#" + idPrefix + "-hidden", reverse: "id", forceMatch: true, - minLength: 1 + minLength: 0 }; } diff --git a/opentreemap/treemap/templates/treemap/field/species_div.html b/opentreemap/treemap/templates/treemap/field/species_div.html index c9e7a898a..5ba9a2a02 100644 --- a/opentreemap/treemap/templates/treemap/field/species_div.html +++ b/opentreemap/treemap/templates/treemap/field/species_div.html @@ -13,6 +13,10 @@
{% include "treemap/field/species_typeahead.html" %} + +
data-href="{{ request.get_full_path }}" class="btn btn-sm btn-danger">{% trans "Delete" %} + {% endif %}

diff --git a/opentreemap/treemap/templates/treemap/partials/map_feature_popup.html b/opentreemap/treemap/templates/treemap/partials/map_feature_popup.html index da2a57ff1..e2d0a1ad5 100644 --- a/opentreemap/treemap/templates/treemap/partials/map_feature_popup.html +++ b/opentreemap/treemap/templates/treemap/partials/map_feature_popup.html @@ -19,6 +19,7 @@

{{ feature.title }}

{% trans "More Details" %} + {% if request.user.is_authenticated %} {{ feature.title }} data-href="{% url 'map_feature_detail_edit' instance_url_name=request.instance.url_name feature_id=feature.pk edit='edit' %}" class="btn btn-sm btn-info">{% trans "Edit" %} + {% endif %} {% if features|length > 1 %} {% empty %} diff --git a/opentreemap/treemap/templates/treemap/partials/upload_file.html b/opentreemap/treemap/templates/treemap/partials/upload_file.html index 87d896ebc..f70389d8f 100644 --- a/opentreemap/treemap/templates/treemap/partials/upload_file.html +++ b/opentreemap/treemap/templates/treemap/partials/upload_file.html @@ -16,6 +16,8 @@

{{ title }}

diff --git a/opentreemap/treemap/urls.py b/opentreemap/treemap/urls.py index 51219fff3..8cc0c7eaf 100644 --- a/opentreemap/treemap/urls.py +++ b/opentreemap/treemap/urls.py @@ -73,6 +73,8 @@ routes.add_tree_photo, name='add_photo_to_plot'), url(r'^plots/(?P\d+)/tree/(?P\d+)/photo$', routes.add_tree_photo, name='add_photo_to_tree'), + url(r'^plots/(?P\d+)/tree/(?P\d+)/photo/label$', + routes.add_tree_photo_with_label, name='add_photo_to_tree_with_label'), url(r'^config/settings.js$', routes.instance_settings_js, name='settings'), diff --git a/opentreemap/treemap/views/misc.py b/opentreemap/treemap/views/misc.py index cbf8b9809..347e77ab2 100644 --- a/opentreemap/treemap/views/misc.py +++ b/opentreemap/treemap/views/misc.py @@ -28,6 +28,7 @@ from treemap.lib.object_caches import udf_defs from treemap.units import get_unit_abbreviation, get_units from treemap.util import leaf_models_of_class +from treemap.images import get_image_from_request _SCSS_VAR_NAME_RE = re.compile('^[_a-zA-Z][-_a-zA-Z0-9]*$') @@ -91,10 +92,21 @@ def get_map_view_context(request, instance): instance, filter_fields=['tree.id', 'tree.species', 'tree.diameter'] ) + add_map_info_to_context(context, instance) return context +def map_save_image_with_label(request, instance, label): + """ + Save an image with this label in the session. + This is needed when a user is creating a new tree from the website + """ + data = get_image_from_request(request) + request.session[label] = data + return + + def add_map_info_to_context(context, instance): all_polygon_types = {c.map_feature_type for c in leaf_models_of_class(PolygonalMapFeature)} diff --git a/opentreemap/treemap/views/tree.py b/opentreemap/treemap/views/tree.py index a70304a85..77b5b3bb4 100644 --- a/opentreemap/treemap/views/tree.py +++ b/opentreemap/treemap/views/tree.py @@ -13,7 +13,7 @@ from django.http import HttpResponseRedirect from treemap.search import Filter -from treemap.models import Tree, Plot +from treemap.models import Tree, Plot, MapFeaturePhotoLabel from treemap.ecobenefits import get_benefits_for_filter from treemap.ecocache import get_cached_plot_count from treemap.lib import format_benefits @@ -27,6 +27,37 @@ def tree_detail(request, instance, feature_id, tree_id): 'feature_id': feature_id})) +def create_map_feature_photo_label(photo, label): + map_feature_photo_label = MapFeaturePhotoLabel() + map_feature_photo_label.map_feature_photo = photo + map_feature_photo_label.name = label + map_feature_photo_label.save() + return map_feature_photo_label + + +def add_tree_photo_with_label(request, instance, feature_id, tree_id=None): + error = None + try: + label = request.POST['label'] + photo, tree = add_tree_photo_helper( + request, instance, feature_id, tree_id) + + create_map_feature_photo_label(photo, label) + + photos = tree.photos() + except ValidationError as e: + trees = Tree.objects.filter(pk=tree_id) + if len(trees) == 1: + photos = trees[0].photos() + else: + photos = [] + # TODO: Better display error messages in the view + error = '; '.join(e.messages) + return {'photos': [context_dict_for_photo(request, photo) + for photo in photos], + 'error': error} + + def add_tree_photo(request, instance, feature_id, tree_id=None): error = None try: From 9edffb9451838e1ffd0b50d6d01c76f8dd5dd29c Mon Sep 17 00:00:00 2001 From: Tom Z Date: Sun, 19 Apr 2020 21:47:33 -0400 Subject: [PATCH 15/71] 20200419_emails_and_embed - fixes and embedding --- opentreemap/manage_treemap/lib/email.py | 5 +---- opentreemap/opentreemap/urls.py | 11 +++++++++-- opentreemap/treemap/js/src/mapPage/browseTreesMode.js | 1 + .../treemap/templates/treemap/map-add-tree.html | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/opentreemap/manage_treemap/lib/email.py b/opentreemap/manage_treemap/lib/email.py index 3241fd84b..fc63fd9ed 100644 --- a/opentreemap/manage_treemap/lib/email.py +++ b/opentreemap/manage_treemap/lib/email.py @@ -18,11 +18,8 @@ def send_email(tmpl, ctxt, recipient_list): from_email = settings.DEFAULT_FROM_EMAIL subject, message = _get_email_subj_and_body(tmpl, ctxt) - connection = get_connection(fail_silently=True) mail = EmailMultiAlternatives(subject, message, - from_email, recipient_list, - connection=connection) + from_email, recipient_list) mail.attach_alternative(message, 'text/html') - return mail.send() diff --git a/opentreemap/opentreemap/urls.py b/opentreemap/opentreemap/urls.py index e94ea4f8c..1029b58bc 100644 --- a/opentreemap/opentreemap/urls.py +++ b/opentreemap/opentreemap/urls.py @@ -28,7 +28,13 @@ # For URLs included via .urls, see /tests # For "top level" URLs defined here, see treemap/tests/urls.py (RootUrlTests) -urlpatterns = [ +root_url = [] +if hasattr(settings, 'DEFAULT_INSTANCE') and settings.DEFAULT_INSTANCE: + root_url.append(url(r'^$', RedirectView.as_view(url='/{}'.format(settings.DEFAULT_INSTANCE)))) +else: + root_url.append(url(r'^$', routes.landing_page)) + +urlpatterns = root_url + [ url(r'^robots.txt$', RedirectView.as_view( url='/static/robots.txt', permanent=True)), # Setting permanent=False in case we want to allow customizing favicons @@ -38,7 +44,7 @@ url('^comments/', include('django_comments.urls')), url(r'^', include('geocode.urls')), url(r'^stormwater/', include('stormwater.urls')), - url(r'^$', routes.landing_page), + #url(r'^$', routes.landing_page), url(r'^config/settings.js$', routes.root_settings_js), url(r'^users/%s/$' % USERNAME_PATTERN, routes.user, name='user'), @@ -80,6 +86,7 @@ url(r'', include('modeling.urls')), ] + if settings.USE_JS_I18N: js_i18n_info_dict = { 'domain': 'djangojs', diff --git a/opentreemap/treemap/js/src/mapPage/browseTreesMode.js b/opentreemap/treemap/js/src/mapPage/browseTreesMode.js index 9dc6e1180..7657bc7df 100644 --- a/opentreemap/treemap/js/src/mapPage/browseTreesMode.js +++ b/opentreemap/treemap/js/src/mapPage/browseTreesMode.js @@ -160,6 +160,7 @@ function makePopup(latLon, html) { var $popup = $(html); if (embed) { $popup.find('a').attr('target', '_blank'); + $popup.find('.popup-btns').css('display', 'none'); } var $popupContents = $($popup[0].outerHTML); diff --git a/opentreemap/treemap/templates/treemap/map-add-tree.html b/opentreemap/treemap/templates/treemap/map-add-tree.html index 7daaf1dcb..4c94e8987 100644 --- a/opentreemap/treemap/templates/treemap/map-add-tree.html +++ b/opentreemap/treemap/templates/treemap/map-add-tree.html @@ -67,7 +67,7 @@

{% trans "Planting Site Information" %}

{% for udf in group.collection_udfs %} - {% with title_prefix=udf.model_type %} + {% with title_prefix="Planting Site" %} {% include "treemap/partials/collectionudf.html" with udf=udf title_prefix=title_prefix model=tree values=values %} {% endwith %} {% endfor %} From 319a1155e1588c700b32c07d9e36733384e4800e Mon Sep 17 00:00:00 2001 From: Tom Z Date: Mon, 20 Apr 2020 21:35:50 -0400 Subject: [PATCH 16/71] 20200419_emails_and_embed - required emails --- .../js/src/lib/uploadPanelAddTreePhoto.js | 5 +++ .../templates/treemap/map-add-tree.html | 31 +++++++++++++------ .../treemap/partials/upload_file.html | 3 +- opentreemap/treemap/views/map_feature.py | 13 ++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/opentreemap/treemap/js/src/lib/uploadPanelAddTreePhoto.js b/opentreemap/treemap/js/src/lib/uploadPanelAddTreePhoto.js index 1cfa7842c..5e2e9d52d 100644 --- a/opentreemap/treemap/js/src/lib/uploadPanelAddTreePhoto.js +++ b/opentreemap/treemap/js/src/lib/uploadPanelAddTreePhoto.js @@ -55,6 +55,11 @@ module.exports.init = function(options) { data.url = url; data.submit(); }); + + var row = $(input.data('row-id')); + row.addClass('bg-success') + + $(input.data('checkbox-id')).prop('checked', true); $panel.modal('hide'); }, progressall: function (e, data) { diff --git a/opentreemap/treemap/templates/treemap/map-add-tree.html b/opentreemap/treemap/templates/treemap/map-add-tree.html index 4c94e8987..e94da73d1 100644 --- a/opentreemap/treemap/templates/treemap/map-add-tree.html +++ b/opentreemap/treemap/templates/treemap/map-add-tree.html @@ -2,9 +2,9 @@ {% load i18n %} {% trans "Add a Photo" as upload_title %} -{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="shape-photo-upload" label="shape" button_id="add-shape" %} -{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="bark-photo-upload" label="bark" button_id="add-bark" %} -{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="leaf-photo-upload" label="leaf" button_id="add-leaf" %} +{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="shape-photo-upload" label="shape" row_id="add-shape" checkbox_id="has-shape-photo" %} +{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="bark-photo-upload" label="bark" row_id="add-bark" checkbox_id="has-bark-photo" %} +{% include "treemap/partials/upload_file.html" with title=upload_title panel_id="leaf-photo-upload" label="leaf" row_id="add-leaf" checkbox_id="has-leaf-photo" %} {% with nsteps=3 %} -

Ecobenefits By Ward

+

Eco Benefits By Ward

- +
+
diff --git a/opentreemap/manage_treemap/views/reports.py b/opentreemap/manage_treemap/views/reports.py index df7ebd4f8..8171d1dd8 100644 --- a/opentreemap/manage_treemap/views/reports.py +++ b/opentreemap/manage_treemap/views/reports.py @@ -14,9 +14,6 @@ def get_reports_data(request, instance, data_set, aggregation_level): - if aggregation_level == 'ward': - aggregation_level = 'main neighborhood' - data_set_funcs = { 'count': get_tree_count, 'species': get_species_count, @@ -156,7 +153,7 @@ def get_ecobenefits(aggregation_level, instance): """ columns = ['Name'] data = [] - boundaries = Boundary.objects.filter(category='Main Neighborhood').all() + boundaries = Boundary.objects.filter(category__iexact=aggregation_level).order_by('name').all() _filter = Filter(None, None, instance) benefits_all, basis_all = get_benefits_for_filter(_filter) diff --git a/opentreemap/treemap/js/src/lib/MapManager.js b/opentreemap/treemap/js/src/lib/MapManager.js index 3b73c03d7..a84b82806 100644 --- a/opentreemap/treemap/js/src/lib/MapManager.js +++ b/opentreemap/treemap/js/src/lib/MapManager.js @@ -290,6 +290,7 @@ MapManager.prototype = { } map.addLayer(basemapMapping[visible]); + var wardLayer = layersLib.createBoundariesbyCategoryTileLayer('Ward'); var mainNeighborhoodLayer = layersLib.createBoundariesbyCategoryTileLayer('Main Neighborhood'); var neighborhoodLayer = layersLib.createBoundariesbyCategoryTileLayer('Neighborhood'); var parksLayer = layersLib.createBoundariesbyCategoryTileLayer('Park'); @@ -300,10 +301,11 @@ MapManager.prototype = { this.layersControl = L.control.layers( basemapMapping, { + 'Wards': wardLayer, 'Main Neighborhoods + LSP': mainNeighborhoodLayer, 'Neighborhoods': neighborhoodLayer, 'Parks': parksLayer, - 'SID': sidLayer, + 'SIDs': sidLayer, 'Parcels': parcelsLayer, 'Zones': namedZonesLayer, }, { From 73a622fd37ef53430bef505f1976273ad1a96d05 Mon Sep 17 00:00:00 2001 From: Tom Z Date: Mon, 17 Aug 2020 14:42:59 -0400 Subject: [PATCH 46/71] develop - fixed a typo in Species and an incorrect grouping --- opentreemap/manage_treemap/js/src/reports.js | 2 +- .../templates/manage_treemap/partials/reports.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opentreemap/manage_treemap/js/src/reports.js b/opentreemap/manage_treemap/js/src/reports.js index 46f6ef9eb..3db6ff4a6 100644 --- a/opentreemap/manage_treemap/js/src/reports.js +++ b/opentreemap/manage_treemap/js/src/reports.js @@ -57,7 +57,7 @@ var url = reverse.roles_endpoint(config.instance.url_name), )(), treeConditionsByNeighborhoodStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'condition', 'ward') + reverse.get_reports_data(config.instance.url_name, 'condition', 'neighborhood') )(), treeConditionsByWardStream = BU.jsonRequest( 'GET', diff --git a/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html b/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html index 3d92a6b38..3b6250537 100644 --- a/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html +++ b/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html @@ -9,7 +9,7 @@

Trees By Ward

-

Trees By Speices

+

Trees By Species

From a0a0d44f6c7820e65b32332d36638efc5bf907dc Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 7 Sep 2020 20:26:09 -0400 Subject: [PATCH 47/71] feature/dashboard, first draft at adding a public dashboard --- opentreemap/manage_treemap/js/src/reports.js | 479 ++++++++++++------ opentreemap/manage_treemap/routes.py | 18 +- .../manage_treemap/dashboard_base.html | 26 + .../manage_treemap/partials/reports.html | 89 ++-- .../templates/manage_treemap/reports.html | 13 +- opentreemap/manage_treemap/views/reports.py | 61 ++- .../treemap/templates/instance_base.html | 4 + 7 files changed, 448 insertions(+), 242 deletions(-) create mode 100644 opentreemap/manage_treemap/templates/manage_treemap/dashboard_base.html diff --git a/opentreemap/manage_treemap/js/src/reports.js b/opentreemap/manage_treemap/js/src/reports.js index 3db6ff4a6..6a59f8b7a 100644 --- a/opentreemap/manage_treemap/js/src/reports.js +++ b/opentreemap/manage_treemap/js/src/reports.js @@ -15,62 +15,99 @@ var $ = require('jquery'), reverse = require('reverse'); var dom = { - selects: 'select[data-name]', - radios: ':radio:checked[data-name]', - roleIds: '[data-roles]', - createNewRole: '#create_new_role', - newRoleName: '#new_role_name', - roles: '#role-info', - edit: '.editBtn', - save: '.saveBtn', - cancel: '.cancelBtn', - addRole: '.addRoleBtn', - addRoleModal: '#add-role-modal', spinner: '.spinner', - rolesTableContainer: '#role-info .role-table-scroll', newFieldsAlert: '#new-fields-alert', newFieldsDismiss: '#new-fields-dismiss', + aggregationLevelDropdown: '#select-aggregation', + + neighborhoodDropdownContainer: '#select-neighborhoods-container', + wardDropdownContainer: '#select-wards-container', + + neighborhoodDropdown: '#select-neighborhoods', + wardDropdown: '#select-wards', + chart: '#group-chart canvas', - treesByNeighborhoodChart: '#trees-by-neighborhood-chart canvas', - treesByWardChart: '#trees-by-ward-chart canvas', + treeCountsChart: '#tree-counts-chart canvas', speciesChart: '#species-chart canvas', - treeConditionsByNeighborhoodChart: '#tree-conditions-by-neighborhood-chart canvas', - treeConditionsByWardChart: '#tree-conditions-by-ward-chart canvas', + treeConditionsChart: '#tree-conditions-chart canvas', treeDiametersChart: '#tree-diameters-chart canvas', + ecobenefitsByWardTableHeader: '#ecobenefits-by-ward-table thead', ecobenefitsByWardTableBody: '#ecobenefits-by-ward-table tbody', ecobenefitsByWardTotal: '#ecobenefits-by-ward-total' }; -var url = reverse.roles_endpoint(config.instance.url_name), - treesByNeighborhoodStream = BU.jsonRequest( - 'GET', - reverse.get_reports_data(config.instance.url_name, 'count', 'neighborhood') - )(), - treesByWardStream = BU.jsonRequest( - 'GET', - reverse.get_reports_data(config.instance.url_name, 'count', 'ward') - )(), - speciesStream = BU.jsonRequest( +var charts = { + treeCountsChart: null, + speciesChart: null, + treeConditionsChart: null, + treeDiametersChart: null, + + ecobenefitsByWardTableHeader: null, + ecobenefitsByWardTableBody: null, + ecobenefitsByWardTotal: null +}; + +// a cache to hold our data +var dataCache = { + treeCountsChart: null, + speciesChart: null, + treeConditionsChart: null, + treeDiametersChart: null, + ecobenefits: null, +}; + +var onValueFunctions = { + treeCountsChart: null, + speciesChart: null, + treeConditionsChart: null, + treeDiametersChart: null, + ecobenefits: null, +} + +var url = reverse.roles_endpoint(config.instance.url_name); + +function loadData() { + + var aggregationLevel = $(dom.aggregationLevelDropdown).val(); + var treeCountStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'species', 'none') - )(), - treeConditionsByNeighborhoodStream = BU.jsonRequest( + reverse.get_reports_data(config.instance.url_name, 'count', aggregationLevel) + )(); + treeCountStream.onError(showError); + treeCountStream.onValue(onValueFunctions.treeCountsChart); + + var speciesStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'condition', 'neighborhood') - )(), - treeConditionsByWardStream = BU.jsonRequest( + reverse.get_reports_data(config.instance.url_name, 'species', aggregationLevel) + )(); + speciesStream.onError(showError); + speciesStream.onValue(onValueFunctions.speciesChart); + + var treeConditionsStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'condition', 'ward') - )(), - ecobenefitsByWardStream = BU.jsonRequest( + reverse.get_reports_data(config.instance.url_name, 'condition', aggregationLevel) + )(); + treeConditionsStream.onError(showError); + treeConditionsStream.onValue(onValueFunctions.treeConditionsChart); + + var treeDiametersStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'ecobenefits', 'ward') - )(), - treeDiametersStream = BU.jsonRequest( + reverse.get_reports_data(config.instance.url_name, 'diameter', aggregationLevel) + )(); + treeDiametersStream.onError(showError); + treeDiametersStream.onValue(onValueFunctions.treeDiametersChart); + + $(dom.ecobenefitsByWardTotal).html(''); + $(dom.spinner).show(); + var ecobenefitsStream = BU.jsonRequest( 'GET', - reverse.get_reports_data(config.instance.url_name, 'diameter', 'none') + reverse.get_reports_data(config.instance.url_name, 'ecobenefits', aggregationLevel) )(); + ecobenefitsStream.onError(showError); + ecobenefitsStream.onValue(onValueFunctions.ecobenefits); +} + function showError(resp) { enableSave(); @@ -78,161 +115,200 @@ function showError(resp) { } var chartColors = { - red: 'rgb(255, 99, 132)', orange: 'rgb(255, 159, 64)', yellow: 'rgb(255, 205, 86)', green: 'rgb(75, 192, 192)', blue: 'rgb(54, 162, 235)', purple: 'rgb(153, 102, 255)', grey: 'rgb(201, 203, 207)', - black: 'rgb(0, 0, 0)' + + // a less saturated red + red: '#8b1002', + + // a softer black + black: '#303031' }; // theme from https://learnui.design/tools/data-color-picker.html -// starting with #add142, which is the lime-green success color in +// starting with #8baa3d, which is the otm-green color in // _base.scss var otmGreen = '#8baa3d'; var otmLimeGreen = '#add142'; var chartColorTheme = [ - '#ffffff', - '#e7f0c2', - '#cce085', - '#add142', - '#6cc259', - '#16b06e', - '#009b7e', - '#008484', - '#006d81', - '#005673', '#003f5c', + '#00506b', + '#006274', + '#007374', + '#00836c', + '#1c935f', + '#59a04e', + '#8baa3d' ]; -treesByNeighborhoodStream.onError(showError); -treesByNeighborhoodStream.onValue(function (results) { - var chart = new Chart($(dom.treesByNeighborhoodChart), { - type: 'bar', - data: { - labels: results['data'].map(x => x['name']), - datasets: [{ - label: 'Trees', - borderColor: otmLimeGreen, - backgroundColor: otmGreen, - data: results['data'].map(x => x['count']) - }] - }, - }); -}); -treesByWardStream.onError(showError); -treesByWardStream.onValue(function (results) { - var chart = new Chart($(dom.treesByWardChart), { - type: 'bar', - data: { - labels: results['data'].map(x => x['name']), - datasets: [{ - label: 'Trees', - borderColor: otmLimeGreen, - backgroundColor: otmGreen, - data: results['data'].map(x => x['count']) - }] - }, - }); -}); +onValueFunctions.treeCountsChart = function (results) { + var data = results['data'] + dataCache.treeCountsChart = data; -speciesStream.onError(showError); -speciesStream.onValue(function (results) { + if (charts.treeCountsChart == null) { + var chart = new Chart($(dom.treeCountsChart), { + type: 'bar', + data: { + labels: [], + datasets: [] + } + }); + + charts.treeCountsChart = chart; + } + + updateTreeCountsData(data); +}; + +function updateTreeCountsData(data) { + var chart = charts.treeCountsChart; + if (chart == null) { + return; + } + + chart.data.labels = data.map(x => x['name']); + chart.data.datasets = [{ + label: 'Trees', + borderColor: otmLimeGreen, + backgroundColor: otmGreen, + data: data.map(x => x['count']) + }]; + chart.update(); +} + +onValueFunctions.speciesChart = function (results) { var data = results['data']; + dataCache.speciesChart = data; + + updateSpeciesData(data); +} + +function updateSpeciesData(data) { + var chart = charts.speciesChart; + if (chart != null) { + chart.destroy(); + } + + // reduce the species and counts, as there are multiple given the aggregation + var reduceFunc = function(acc, value) { + acc[value['species_name']] = acc[value['species_name']] + value['count'] + || value['count']; + return acc; + } + var dataObj = data.reduce(reduceFunc, {}); + // make into a list of items and sort descending + data = Object.keys(dataObj).map(k => {return {name: k, count: dataObj[k]}}) + .sort((first, second) => second['count'] - first['count']); + + // take the first N and aggregate the rest + var finalData = data.slice(0, 5); + var otherSum = data.slice(5).reduce((acc, val) => acc + val['count'], 0); + finalData.push({name: 'Other', count: otherSum}) + var chart = new Chart($(dom.speciesChart), { type: 'pie', data: { - labels: data.map(x => x['name']), + labels: finalData.map(x => x['name']), datasets: [{ - data: data.map(x => x['count']), - backgroundColor: data.map((x, i) => chartColorTheme[i]), + data: finalData.map(x => x['count']), + backgroundColor: finalData.map((x, i) => chartColorTheme[i]), borderColor: 'rgba(200, 200, 200, 0.75)', hoverBorderColor: 'rgba(200, 200, 200, 1)', }] - }, + } }); -}); + charts.speciesChart = chart; + chart.update(); +} -treeConditionsByNeighborhoodStream.onError(showError); -treeConditionsByNeighborhoodStream.onValue(function (results) { +onValueFunctions.treeConditionsChart = function (results) { var data = results['data']; - var chart = new Chart($(dom.treeConditionsByNeighborhoodChart), { - type: 'bar', - options: { - scales: { - xAxes: [{ - stacked: true, - }], - yAxes: [{ - stacked: true - }] - } - }, - data: { - labels: data.map(x => x['name']), - datasets: [ - { - label: 'Healthy', - data: data.map(x => x['healthy']), - backgroundColor: otmLimeGreen - }, - { - label: 'Unhealthy', - data: data.map(x => x['unhealthy']), - backgroundColor: chartColors.red - }, - { - label: 'Dead', - data: data.map(x => x['dead']), - backgroundColor: chartColors.black - }] - }, - }); -}); + dataCache.treeConditionsChart = data; -treeConditionsByWardStream.onError(showError); -treeConditionsByWardStream.onValue(function (results) { - var data = results['data']; - var chart = new Chart($(dom.treeConditionsByWardChart), { - type: 'bar', - options: { - scales: { - xAxes: [{ - stacked: true, - }], - yAxes: [{ - stacked: true - }] + if (charts.treeConditionsChart == null) { + var chart = new Chart($(dom.treeConditionsChart), { + type: 'bar', + options: { + scales: { + xAxes: [{ + stacked: true, + }], + yAxes: [{ + stacked: true + }] + } + }, + data: { + labels: [], + datasets: [] } + }); + charts.treeConditionsChart = chart; + } + + updateTreeConditionsChart(data); +} + +function updateTreeConditionsChart(data) { + var chart = charts.treeConditionsChart; + if (chart == null) { + return; + } + + chart.data.labels = data.map(x => x['name']); + chart.data.datasets = [ + { + label: 'Healthy', + data: data.map(x => x['healthy']), + backgroundColor: otmGreen }, - data: { - labels: data.map(x => x['name']), - datasets: [ - { - label: 'Healthy', - data: data.map(x => x['healthy']), - backgroundColor: otmLimeGreen - }, - { - label: 'Unhealthy', - data: data.map(x => x['unhealthy']), - backgroundColor: chartColors.red - }, - { - label: 'Dead', - data: data.map(x => x['dead']), - backgroundColor: chartColors.black - }] + { + label: 'Unhealthy', + data: data.map(x => x['unhealthy']), + backgroundColor: chartColors.red }, - }); -}); + { + label: 'Dead', + data: data.map(x => x['dead']), + backgroundColor: chartColors.black + } + ]; + chart.update(); +} -treeDiametersStream.onError(showError); -treeDiametersStream.onValue(function (results) { +onValueFunctions.treeDiametersChart = function (results) { var data = results['data']; + dataCache.treeDiametersChart = data; + updateTreeDiametersChart(data); +} + +function updateTreeDiametersChart(data) { + var chart = charts.treeDiametersChart; + if (chart != null) { + chart.destroy(); + } + // + // reduce the species and counts, as there are multiple given the aggregation + var reduceFunc = function(acc, value) { + var diameter = value['diameter']; + if (diameter <= 5) { + acc['< 5 in.'] = acc['< 5 in.'] + 1 || 1; + } else if (diameter > 5 && diameter < 25){ + acc['> 5 in. and < 25 in.'] = acc['> 5 in. and < 25 in.'] + 1 || 1; + } else { + acc['> 25 in.'] = acc['> 25 in.'] + 1 || 1; + } + return acc; + } + var dataObj = data.reduce(reduceFunc, {}); + // make into a list of items and sort descending + data = Object.keys(dataObj).map(k => {return {name: k, count: dataObj[k]}}); + var colors = chartColorTheme.reverse(); var chart = new Chart($(dom.treeDiametersChart), { type: 'pie', @@ -240,17 +316,23 @@ treeDiametersStream.onValue(function (results) { labels: data.map(x => x['name']), datasets: [{ data: data.map(x => x['count']), - backgroundColor: data.map((x, i) => colors[i]), + backgroundColor: data.map((x, i) => colors[i * 2]), borderColor: 'rgba(200, 200, 200, 0.75)', hoverBorderColor: 'rgba(200, 200, 200, 1)', }] }, }); -}); + charts.treeDiametersChart = chart; +} -ecobenefitsByWardStream.onError(showError); -ecobenefitsByWardStream.onValue(function (results) { +onValueFunctions.ecobenefits = function (results) { var data = results['data']; + dataCache.ecobenefits = data; + $(dom.spinner).hide(); + updateEcobenefits(data); +} + +function updateEcobenefits(data) { var columns = data['columns']; var columnHtml = '' + columns.map(x => '' + x + '').join('') + ''; var dataHtml = data['data'].map(row => '' + row.map((x, i) => { @@ -280,7 +362,7 @@ ecobenefitsByWardStream.onValue(function (results) { minimumFractionDigits: 2, maximumFractionDigits: 2 }) + '
'); -}); +} function formatColumn(column, columnName) { if (column == null) @@ -299,13 +381,68 @@ function formatColumn(column, columnName) { return column; } +$(dom.neighborhoodDropdown).change(function(event) { + var name = $(this).val(); + filterDataByAggregation(name); +}); -buttonEnabler.run(); -U.modalsFocusOnFirstInputWhenShown(); -$(dom.addRole).on('click', function () { - $(dom.addRoleModal).modal('show'); +$(dom.wardDropdown).change(function(event) { + var name = $(this).val(); + filterDataByAggregation(name); }); +function filterDataByAggregation(name) { + var data = dataCache.treeCountsChart; + + updateTreeCountsData( + data.filter(x => name == 'all' || name.includes(x['name'])) + ); + + data = dataCache.speciesChart; + updateSpeciesData( + data.filter(x => name == 'all' || name.includes(x['name'])) + ); + + data = dataCache.treeConditionsChart; + updateTreeConditionsChart( + data.filter(x => name == 'all' || name.includes(x['name'])) + ); + + data = dataCache.treeDiametersChart; + updateTreeDiametersChart( + data.filter(x => name == 'all' || name.includes(x['name'])) + ); + + data = dataCache.ecobenefits; + updateEcobenefits({ + columns: data['columns'], + data: data['data'].filter(x => name == 'all' || name.includes(x[0])) + }); +} + +$(dom.aggregationLevelDropdown).change(function(event) { + var aggregationLevel = $(dom.aggregationLevelDropdown).val(); + + $(dom.wardDropdownContainer + " option:selected").removeAttr("selected"); + $(dom.neighborhoodDropdownContainer + " option:selected").removeAttr("selected"); + + // could probably do toggle, but i'm paranoid something will break + if (aggregationLevel == "ward") { + $(dom.wardDropdownContainer).show(); + $(dom.wardDropdownContainer + " option[value=all]").attr("selected", true); + $(dom.neighborhoodDropdownContainer).hide(); + } else { + $(dom.wardDropdownContainer).hide(); + $(dom.neighborhoodDropdownContainer).show(); + $(dom.neighborhoodDropdownContainer + " option[value=all]").attr("selected", true); + } + + loadData(); +}); + + +buttonEnabler.run(); +U.modalsFocusOnFirstInputWhenShown(); var alertDismissStream = $(dom.newFieldsDismiss).asEventStream('click') .doAction('.preventDefault') @@ -314,8 +451,12 @@ var alertDismissStream = $(dom.newFieldsDismiss).asEventStream('click') alertDismissStream.onValue(function() { $(dom.newFieldsAlert).hide(); - $(dom.roles).find('tr.active').removeClass('active'); }); adminPage.init(Bacon.mergeAll(alertDismissStream)); +// initially, show by Ward +$(dom.wardDropdownContainer).show(); +$(dom.neighborhoodDropdownContainer).hide(); +$(dom.wardDropdownContainer + " option[value=all]").attr("selected", true); +loadData(); diff --git a/opentreemap/manage_treemap/routes.py b/opentreemap/manage_treemap/routes.py index f71d71379..7bf138252 100644 --- a/opentreemap/manage_treemap/routes.py +++ b/opentreemap/manage_treemap/routes.py @@ -19,7 +19,7 @@ from manage_treemap.views import update_instance_fields_with_validator from manage_treemap.views.roles import roles_list, roles_update, roles_create from manage_treemap.views.groups import groups_list, get_groups_data -from manage_treemap.views.reports import get_reports_data +from manage_treemap.views.reports import get_reports_data, reports from manage_treemap.views.udf import (udf_bulk_update, udf_create, udf_list, udf_delete_popup, udf_delete, udf_update_choice, @@ -28,7 +28,7 @@ user_roles_list, update_user_roles, create_user_role, remove_invited_user_from_instance) from treemap.decorators import (require_http_method, admin_instance_request, - return_400_if_validation_errors) + return_400_if_validation_errors, instance_request) admin_route = lambda **kwargs: admin_instance_request(route(**kwargs)) @@ -223,9 +223,11 @@ admin_instance_request, get_groups_data) - -reports = admin_route( - GET=do(render_template('manage_treemap/reports.html'), groups_list), +reports = do( + instance_request, + route( + GET=do(render_template('manage_treemap/reports.html'), reports), + ) #PUT=do(groups_update), #POST=do(render_template('manage_treemap/partials/groups.html'), # groups_create) @@ -234,5 +236,7 @@ # FIXME move to API get_reports_data = do( json_api_call, - admin_instance_request, - get_reports_data) + instance_request, + #admin_instance_request, + get_reports_data +) diff --git a/opentreemap/manage_treemap/templates/manage_treemap/dashboard_base.html b/opentreemap/manage_treemap/templates/manage_treemap/dashboard_base.html new file mode 100644 index 000000000..09f413fe7 --- /dev/null +++ b/opentreemap/manage_treemap/templates/manage_treemap/dashboard_base.html @@ -0,0 +1,26 @@ +{% extends "instance_base.html" %} +{% load l10n %} +{% load i18n %} +{% load auth_extras %} +{% load instance_config %} +{% load is_current_view %} + +{% block page_title %} | {% trans "Dashboard" %} {% endblock %} + +{% block activeexplore %} +{% endblock %} + +{% block activemanagement %} +{% endblock %} + +{% block activedashboard %} + active +{% endblock %} + +{% block header %} +{% endblock header %} +{% block subhead %} +{% endblock subhead %} + +{% block content %} +{% endblock content %} diff --git a/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html b/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html index 3b6250537..93807f6b1 100644 --- a/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html +++ b/opentreemap/manage_treemap/templates/manage_treemap/partials/reports.html @@ -1,38 +1,67 @@ -
-

Trees By Neighborhood

-
- -
+
+
+

Trees Counts

+
+ +
-

Trees By Ward

-
- -
+

Trees By Species

+
+ +
-

Trees By Species

-
- -
+

Tree Conditions

+
+ +
-

Tree Conditions By Neighborhood

-
- -
+

Tree Diameters

+
+ +
-

Tree Conditions By Ward

-
- +

Eco Benefits

+ + + + +
+
+

* Total trees based on trees used for Eco Benefits calculator

-

Tree Diameters By Species

-
- -
+
diff --git a/opentreemap/manage_treemap/templates/manage_treemap/reports.html b/opentreemap/manage_treemap/templates/manage_treemap/reports.html index 0289c824b..c965adab9 100644 --- a/opentreemap/manage_treemap/templates/manage_treemap/reports.html +++ b/opentreemap/manage_treemap/templates/manage_treemap/reports.html @@ -1,18 +1,13 @@ -{% extends "manage_treemap/management_base.html" %} +{% extends "manage_treemap/dashboard_base.html" %} {% load render_bundle from webpack_loader %} {% load i18n %} {% load instance_config %} -{% block admin_title %}{% trans "Reports" %}{% endblock %} - -{% block tab_content %} -
+{% block content %} +
diff --git a/opentreemap/manage_treemap/views/reports.py b/opentreemap/manage_treemap/views/reports.py index 8171d1dd8..f3d782158 100644 --- a/opentreemap/manage_treemap/views/reports.py +++ b/opentreemap/manage_treemap/views/reports.py @@ -10,7 +10,18 @@ from treemap.ecobenefits import get_benefits_for_filter from treemap.search import Filter -from treemap.models import Boundary +from treemap.models import Boundary, NeighborhoodGroup + + +def reports(request, instance): + neighborhoods = Boundary.objects.filter(category__iexact='neighborhood').order_by('name').all() + wards = Boundary.objects.filter(category__iexact='ward').order_by('name').all() + + return { + 'instance': instance, + 'neighborhoods': neighborhoods, + 'wards': wards, + } def get_reports_data(request, instance, data_set, aggregation_level): @@ -49,31 +60,21 @@ def get_tree_count(aggregation_level, instance): def get_species_count(aggregation_level, instance): query = """ - with top_ten as ( - SELECT s.common_name as name, - count(1) as "count" - from treemap_mapfeature m - join treemap_tree t on m.id = t.plot_id - left join treemap_species s on s.id = t.species_id - WHERE 1=1 - and s.common_name is not null - group by s.common_name - order by count(1) desc - limit 10 - ) - SELECT top_ten.name, top_ten.count from top_ten - - union all - - SELECT 'Other' as "name", + SELECT s.common_name as species_name, + b.name as name, count(1) as "count" from treemap_mapfeature m join treemap_tree t on m.id = t.plot_id left join treemap_species s on s.id = t.species_id + left join treemap_boundary b on + (st_within(m.the_geom_webmercator, b.the_geom_webmercator)) WHERE 1=1 - and s.common_name not in (select name from top_ten) + and s.common_name is not null + and b.name is not null + and lower(b.category) = %s + group by s.common_name, b.name """ - columns = ['name', 'count'] + columns = ['species_name', 'name', 'count'] with connection.cursor() as cursor: cursor.execute(query, [aggregation_level]) results = cursor.fetchall() @@ -113,8 +114,8 @@ def get_tree_conditions(aggregation_level, instance): def get_tree_diameters(aggregation_level, instance): query = """ with tstats as ( - select min(diameter) as min, - max(diameter) as max + select min(diameter) as min, + max(diameter) as max from treemap_mapfeature m left join treemap_tree t on m.id = t.plot_id left join treemap_species s on s.id = t.species_id @@ -124,21 +125,25 @@ def get_tree_diameters(aggregation_level, instance): -- this is otherwise probably just wrong data and diameter >= 2.5 ) - select 'Min: ' || trunc(min(diameter)) || '; Max: ' || trunc(max(diameter)) as name, - count(1) as "count" + select diameter as diameter, + b.name as name, + tstats.min as "min", + tstats.max as "max" from treemap_mapfeature m cross join tstats left join treemap_tree t on m.id = t.plot_id left join treemap_species s on s.id = t.species_id + left JOIN treemap_boundary b on (ST_Within(m.the_geom_webmercator, b.the_geom_webmercator)) where 1=1 + and lower(b.category) = %s + and b.name is not null and diameter is not null and s.common_name is not null -- this is otherwise probably just wrong data and diameter >= 2.5 - group by width_bucket(diameter, min, max, 9) - order by width_bucket(diameter, min, max, 9) """ - columns = ['name', 'count'] + + columns = ['diameter', 'name', 'min', 'max'] with connection.cursor() as cursor: cursor.execute(query, [aggregation_level]) results = cursor.fetchall() @@ -165,6 +170,8 @@ def get_ecobenefits(aggregation_level, instance): columns.append('Total Trees') data_all.append(basis_all['plot']['n_objects_used']) + boundary_filters = [] + # iterate over the boundaries and get the benefits for each one for boundary in boundaries: # for now, skip this one diff --git a/opentreemap/treemap/templates/instance_base.html b/opentreemap/treemap/templates/instance_base.html index fbb5a5c32..9898daa47 100644 --- a/opentreemap/treemap/templates/instance_base.html +++ b/opentreemap/treemap/templates/instance_base.html @@ -87,6 +87,10 @@ {% trans "Manage" %} {% endif %} + + {% endblock instancetopnav %} - + {% endblock topnav %} {% block header %} @@ -198,7 +204,7 @@
{% if not embed %} -
{% block footer %}{% endblock footer %}
+
{% block footer %}{% endblock footer %}
{% endif %} {% block config_scripts %} @@ -219,7 +225,7 @@ {% js_reverse_inline %} {% endif %} - {% render_bundle 'js/treemap/base' 'js' %} + {% render_bundle 'js/treemap/base-chunk' 'js' %} {% endblock global_scripts %} {% block templates %} diff --git a/opentreemap/treemap/templates/instance_base.html b/opentreemap/treemap/templates/instance_base.html index 9898daa47..d6da868c8 100644 --- a/opentreemap/treemap/templates/instance_base.html +++ b/opentreemap/treemap/templates/instance_base.html @@ -37,17 +37,18 @@ {% block instancetopnav %} {% if request.instance|feature_enabled:'add_plot' and last_effective_instance_user %} {% if request.instance.has_resources %} - {% else %} -
  • - + - + - + + + {{ request.instance.name }} + + +
  • + + {% if last_effective_instance_user|has_permission:'modeling' %} - {% endif %} {% if last_effective_instance_user.admin %} - {% endif %} - {% endblock instancetopnav %} + + + {% endif %} {% if error %} {# The data-photo-upload-failed attribute must appear on an element if there was a validation error #} diff --git a/opentreemap/treemap/templates/treemap/partials/step_controls.html b/opentreemap/treemap/templates/treemap/partials/step_controls.html index 81852a7ab..91d426020 100644 --- a/opentreemap/treemap/templates/treemap/partials/step_controls.html +++ b/opentreemap/treemap/templates/treemap/partials/step_controls.html @@ -9,14 +9,24 @@ {% trans "of" %} {{ nsteps }} -
      +
        {% if not first %} - {% endif %} -
      • + + + {% trans "Step" %} + + + {% trans "of" %} + {{ nsteps }} + +
      • + diff --git a/opentreemap/treemap/templates/treemap/partials/step_set_location.html b/opentreemap/treemap/templates/treemap/partials/step_set_location.html index 27e01e969..2d01b4a80 100644 --- a/opentreemap/treemap/templates/treemap/partials/step_set_location.html +++ b/opentreemap/treemap/templates/treemap/partials/step_set_location.html @@ -2,7 +2,7 @@
        {% trans "Set the" %} {{ feature_name }}'s {% trans "location" %} - × + ×
        {# Making onsubmit return false prevents the form from being submitted. We want to run JS code instead #} @@ -12,21 +12,21 @@ {% trans "Use Current Location" %}
        - + {% trans 'Tap on the map, enter an address, or select "Use Current Location."' %} -
        -
    diff --git a/opentreemap/treemap/templates/treemap/map_feature_detail.html b/opentreemap/treemap/templates/treemap/map_feature_detail.html index fc085ca24..e43b2ac8a 100644 --- a/opentreemap/treemap/templates/treemap/map_feature_detail.html +++ b/opentreemap/treemap/templates/treemap/map_feature_detail.html @@ -107,27 +107,26 @@
    -
    -
    +
    +
    {% include map_feature_partial %} -
    - -
    -
    + +
    +
    +
    + {% if feature.is_editable %} + + + {% endif %} +
    +
    - {{ feature.is_editable }} - {% if feature.is_editable %} - - - {% endif %} -
    -
    diff --git a/opentreemap/treemap/templates/treemap/partials/map_feature_detail_base.html b/opentreemap/treemap/templates/treemap/partials/map_feature_detail_base.html index 3058a7094..731783421 100644 --- a/opentreemap/treemap/templates/treemap/partials/map_feature_detail_base.html +++ b/opentreemap/treemap/templates/treemap/partials/map_feature_detail_base.html @@ -7,7 +7,7 @@ {% load auth_extras %} {% load comment_sequence %} -
    +
    {% if request.instance.is_public %}