From 5723cb1fea490d885778fba7cff220d2bf0cfad8 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Tue, 12 Dec 2023 12:30:39 -0600 Subject: [PATCH 01/27] WIP analytics page in Admin --- src/registrar/admin.py | 59 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 12 ++++ 2 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 src/registrar/templates/admin/analytics.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c5f5be276..2156aeeb1 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,15 +1,17 @@ import logging +import datetime + from django import forms -from django.db.models.functions import Concat +from django.db.models.functions import Concat, Avg, F from django.http import HttpResponse -from django.shortcuts import redirect +from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType -from django.http.response import HttpResponseRedirect -from django.urls import reverse +from django.http.response import HttpResponse, HttpResponseRedirect +from django.urls import path, reverse from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.domain import Domain from registrar.models.user import User @@ -353,6 +355,39 @@ class MyUserAdmin(BaseUserAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] + def get_urls(self): + urlpatterns = super().get_urls() + + my_urls = [ + path( + "analytics/", + self.admin_site.admin_view(self.user_analytics), + name="user_analytics", + ), + ] + + return my_urls + urlpatterns + + def user_analytics(self, request): + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + context = dict( + **self.admin_site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + ), + ) + return render(request, "admin/analytics.html", context) + # Let's define First group # (which should in theory be the ONLY group) def group(self, obj): @@ -1095,8 +1130,6 @@ def export_data_federal(self, request): return response def get_urls(self): - from django.urls import path - urlpatterns = super().get_urls() # Used to extrapolate a path name, for instance @@ -1178,9 +1211,11 @@ def do_delete_domain(self, request, obj): else: self.message_user( request, - "Error deleting this Domain: " - f"Can't switch from state '{obj.state}' to 'deleted'" - ", must be either 'dns_needed' or 'on_hold'", + ( + "Error deleting this Domain: " + f"Can't switch from state '{obj.state}' to 'deleted'" + ", must be either 'dns_needed' or 'on_hold'" + ), messages.ERROR, ) except Exception: @@ -1192,7 +1227,7 @@ def do_delete_domain(self, request, obj): else: self.message_user( request, - ("Domain %s has been deleted. Thanks!") % obj.name, + "Domain %s has been deleted. Thanks!" % obj.name, ) return HttpResponseRedirect(".") @@ -1234,7 +1269,7 @@ def do_place_client_hold(self, request, obj): else: self.message_user( request, - ("%s is in client hold. This domain is no longer accessible on the public internet.") % obj.name, + "%s is in client hold. This domain is no longer accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") @@ -1263,7 +1298,7 @@ def do_remove_client_hold(self, request, obj): else: self.message_user( request, - ("%s is ready. This domain is accessible on the public internet.") % obj.name, + "%s is ready. This domain is accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html new file mode 100644 index 000000000..2906af223 --- /dev/null +++ b/src/registrar/templates/admin/analytics.html @@ -0,0 +1,12 @@ +{% extends "admin/base_site.html" %} + +{% block content_title %}Registrar Analytics{% endblock %} + +{% block content %} + +{% endblock %} From 348d77726035f73e4cd817429e741b4439aa39f0 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Mon, 5 Feb 2024 13:22:17 -0500 Subject: [PATCH 02/27] fix django function imports --- src/registrar/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 2156aeeb1..aa9795eac 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2,7 +2,8 @@ import datetime from django import forms -from django.db.models.functions import Concat, Avg, F +from django.db.models import Avg, F +from django.db.models.functions import Concat from django.http import HttpResponse from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions From 56cd0b6d156f0270f78daf8150e8d79fee1a1b33 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 27 Feb 2024 18:43:59 -0500 Subject: [PATCH 03/27] Gather all existing reports on analytics page --- src/registrar/admin.py | 90 +++++++++---------- src/registrar/assets/sass/_theme/_admin.scss | 3 +- src/registrar/templates/admin/analytics.html | 63 +++++++++++-- src/registrar/templates/admin/index.html | 33 ------- .../django/admin/domain_change_list.html | 23 ----- 5 files changed, 99 insertions(+), 113 deletions(-) delete mode 100644 src/registrar/templates/admin/index.html delete mode 100644 src/registrar/templates/django/admin/domain_change_list.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index aa9795eac..94b8e3dfd 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -357,17 +357,58 @@ class MyUserAdmin(BaseUserAdmin): ordering = ["first_name", "last_name", "email"] def get_urls(self): + """Map a new page in admin for analytics.""" urlpatterns = super().get_urls() + # Used to extrapolate a path name, for instance + # name="{app_label}_{model_name}_export_data_type" + domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name + my_urls = [ path( "analytics/", self.admin_site.admin_view(self.user_analytics), name="user_analytics", ), + path( + "export_data_type/", + self.export_data_type, + name="%s_%s_export_data_type" % domain_path_meta, + ), + path( + "export_data_full/", + self.export_data_full, + name="%s_%s_export_data_full" % domain_path_meta, + ), + path( + "export_data_federal/", + self.export_data_federal, + name="%s_%s_export_data_federal" % domain_path_meta, + ), ] return my_urls + urlpatterns + + def export_data_type(self, request): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + + def export_data_full(self, request): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + + def export_data_federal(self, request): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response def user_analytics(self, request): last_30_days_applications = models.DomainApplication.objects.filter( @@ -1103,60 +1144,11 @@ def organization_type(self, obj): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def get_urls(self): - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - info = self.model._meta.app_label, self.model._meta.model_name - - my_url = [ - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % info, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % info, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % info, - ), - ] - - return my_url + urlpatterns - def response_change(self, request, obj): # Create dictionary of action functions ACTION_FUNCTIONS = { diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 760c4f13a..04dceef08 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -112,7 +112,8 @@ html[data-theme="light"] { .change-list .usa-table thead th, body.dashboard, body.change-list, - body.change-form { + body.change-form, + .analytics { color: var(--body-fg); } } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2906af223..82081d629 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,12 +1,61 @@ {% extends "admin/base_site.html" %} -{% block content_title %}Registrar Analytics{% endblock %} + + +{% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} -
    -
  • User Count: {{ data.user_count }}
  • -
  • Domain Count: {{ data.domain_count }}
  • -
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
  • -
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
  • -
+ + {% block object-tools %} + + {% endblock %} + +
+
+

At a glance

+
+
    +
  • User Count: {{ data.user_count }}
  • +
  • Domain Count: {{ data.domain_count }}
  • +
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
  • +
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
  • +
+
+
+ +
+

Domain growth

+
+ {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + The challenge is in the path definition in urls. It does NOT like admin/export_data/ + + See the commit "Review for ticket #999" + {% endcomment %} +
+
+ + +
+
+ + +
+ +
+
+
+ +
{% endblock %} diff --git a/src/registrar/templates/admin/index.html b/src/registrar/templates/admin/index.html deleted file mode 100644 index 04601ef32..000000000 --- a/src/registrar/templates/admin/index.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends "admin/index.html" %} - -{% block content %} -
- {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} -
-

Reports

-

Domain growth report

- - {% comment %} - Inputs of type date suck for accessibility. - We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. Itdoes NOT like admin/export_data/ - - See the commit "Review for ticket #999" - {% endcomment %} - -
-
- - -
-
- - -
- - -
- -
-
-{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html deleted file mode 100644 index 22df74685..000000000 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "admin/change_list.html" %} - -{% block object-tools %} - - -{% endblock %} \ No newline at end of file From 0a174e5d29b9625c0d2454475b67dc8519656541 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 13:58:33 -0500 Subject: [PATCH 04/27] Reports, chart wip --- src/registrar/admin.py | 143 ++++++----- src/registrar/assets/js/get-gov-admin.js | 28 ++- src/registrar/assets/sass/_theme/_admin.scss | 17 ++ src/registrar/config/settings.py | 2 +- src/registrar/config/urls.py | 42 +++- src/registrar/signals.py | 2 + src/registrar/templates/admin/analytics.html | 105 ++++++-- src/registrar/templates/admin/base_site.html | 2 + src/registrar/tests/test_admin_views.py | 2 +- src/registrar/tests/test_reports.py | 2 +- src/registrar/utility/csv_export.py | 237 +++++++++++++++++-- src/registrar/views/admin_views.py | 58 ++++- 12 files changed, 499 insertions(+), 141 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 34270584a..9bc77b029 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -27,6 +27,7 @@ from django.utils.safestring import mark_safe from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from django.utils.translation import gettext_lazy as _ @@ -363,6 +364,75 @@ class UserContactInline(admin.StackedInline): model = models.Contact +def user_analytics(request): + + end_date = datetime.datetime.today() + start_date = datetime.datetime.today() - datetime.timedelta(days=30) + + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=start_date + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] + + logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # get number of ready domains, counts by org type and election office + # add to context + + # get number of submitted request counts by org type and election office + # add to context + + context = dict( + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ), + ) + return render(request, "admin/analytics.html", context) + class MyUserAdmin(BaseUserAdmin): """Custom user admin class to use our inlines.""" @@ -464,79 +534,6 @@ class Meta: # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] - def get_urls(self): - """Map a new page in admin for analytics.""" - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name - - my_urls = [ - path( - "analytics/", - self.admin_site.admin_view(self.user_analytics), - name="user_analytics", - ), - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % domain_path_meta, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % domain_path_meta, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % domain_path_meta, - ), - ] - - return my_urls + urlpatterns - - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def user_analytics(self, request): - last_30_days_applications = models.DomainApplication.objects.filter( - created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) - ) - avg_approval_time = last_30_days_applications.annotate( - approval_time=F("approved_domain__created_at") - F("created_at") - ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) - context = dict( - **self.admin_site.each_context(request), - data=dict( - user_count=models.User.objects.all().count(), - domain_count=models.Domain.objects.all().count(), - applications_last_30_days=last_30_days_applications.count(), - average_application_approval_time_last_30_days=avg_approval_time, - ), - ) - return render(request, "admin/analytics.html", context) def get_search_results(self, request, queryset, search_term): """ Override for get_search_results. This affects any upstream model using autocomplete_fields, diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ff73acb65..618cc284c 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -322,23 +322,25 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, // Default the value of the end date input field to the current date let endDateInput =document.getElementById('end'); - let exportGrowthReportButton = document.getElementById('exportLink'); + let exportButtons = document.querySelectorAll('.exportLink'); - if (exportGrowthReportButton) { + if (exportButtons.length > 0) { startDateInput.value = currentDate; endDateInput.value = currentDate; - exportGrowthReportButton.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = document.getElementById('exportLink').dataset.exportUrl; - - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; + exportButtons.forEach((btn) => { + btn.addEventListener('click', function() { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); }); } diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 0d232ff41..c74daf678 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -303,3 +303,20 @@ input.admin-confirm-button { display: contents !important; } } + +.usa-button-group { + margin-left: -0.25rem!important; + padding-left: 0!important; + .usa-button-group__item { + list-style-type: none; + line-height: normal; + } + .button { + display: inline-block; + padding: 10px 8px; + line-height: normal; + } + .usa-icon { + top: 2px; + } +} diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index bb8e22ad7..56f3c2090 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -330,7 +330,7 @@ # Google analytics requires that we relax our otherwise # strict CSP by allowing scripts to run from their domain # and inline with a nonce, as well as allowing connections back to their domain -CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] +CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 4bd7b4baf..a9fee650e 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,9 +9,8 @@ from django.views.generic import RedirectView from registrar import views - -from registrar.views.admin_views import ExportData - +from registrar.admin import user_analytics +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from registrar.views.application import Step from registrar.views.utility import always_404 @@ -52,7 +51,42 @@ "admin/logout/", RedirectView.as_view(pattern_name="logout", permanent=False), ), - path("export_data/", ExportData.as_view(), name="admin_export_data"), + path( + "admin/analytics/export_data_type/", + ExportDataType.as_view(), + name="export_data_type", + ), + path( + "admin/analytics/export_data_full/", + ExportDataFull.as_view(), + name="export_data_full", + ), + path( + "admin/analytics/export_data_federal/", + ExportDataFederal.as_view(), + name="export_data_federal", + ), + path( + "admin/analytics/export_domain_growth/", + ExportDataDomainGrowth.as_view(), + name="export_domain_growth", + ), + path( + "admin/analytics/export_managed_unmanaged/", + ExportDataManagedVsUnmanaged.as_view(), + name="export_managed_unmanaged", + ), + path( + "admin/analytics/export_requests/", + ExportDataRequests.as_view(), + name="export_requests", + ), + path( + "admin/analytics/", + admin.site.admin_view(user_analytics), + name="user_analytics", + ), + path("admin/", admin.site.urls), path( "application//edit/", diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 4e7768ef4..ef09e605b 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,6 +27,7 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") + logger.info(f'in handle_profile first {instance}') is_new_user = kwargs.get("created", False) @@ -36,6 +37,7 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact + logger.info(f'inside no matching contacts for first {first_name} last {last_name} email {email}') Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 82081d629..f65aa77cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,25 +1,11 @@ {% extends "admin/base_site.html" %} - +{% load static %} {% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} - {% block object-tools %} - - {% endblock %} -

At a glance

@@ -34,17 +20,46 @@

At a glance

-

Domain growth

+

Current domains

+ +
+ +
+

Growth reports

{% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_data/ + The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ See the commit "Review for ticket #999" {% endcomment %} -
-
+
+
@@ -52,8 +67,58 @@

Domain growth

-
+ +
    +
  • + +
  • +
  • + +
  • +
  • + +
  • +
  • + +
  • +
+ +
+
+ +
+
+ +
+
+
diff --git a/src/registrar/templates/admin/base_site.html b/src/registrar/templates/admin/base_site.html index 73e9ba1f0..58843421a 100644 --- a/src/registrar/templates/admin/base_site.html +++ b/src/registrar/templates/admin/base_site.html @@ -20,7 +20,9 @@ > + + {% endblock %} {% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index aa150d55c..e55175db9 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -26,7 +26,7 @@ def test_export_data_view(self): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 011c60b93..c00c2b221 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -542,7 +542,7 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): are pulled when the growth report conditions are applied to export_domains_to_writed. Test that ready domains are sorted by first_ready/deleted dates first, names second. - We considered testing export_data_growth_to_csv which calls write_body + We considered testing export_data_domain_growth_to_csv which calls write_body and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 90e80f551..1764536b5 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,6 +2,7 @@ import logging from datetime import datetime from registrar.models.domain import Domain +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from django.utils import timezone from django.core.paginator import Paginator @@ -19,10 +20,8 @@ def write_header(writer, columns): Receives params from the parent methods and outputs a CSV with a header row. Works with write_header as long as the same writer object is passed. """ - writer.writerow(columns) - def get_domain_infos(filter_condition, sort_fields): domain_infos = ( DomainInformation.objects.select_related("domain", "authorizing_official") @@ -43,7 +42,6 @@ def get_domain_infos(filter_condition, sort_fields): ) return domain_infos_cleaned - def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" @@ -104,7 +102,6 @@ def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None row = [FIELDS.get(column, "") for column in columns] return row - def _get_security_emails(sec_contact_ids): """ Retrieve security contact emails for the given security contact IDs. @@ -126,7 +123,6 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict - def update_columns_with_domain_managers(columns, max_dm_count): """ Update the columns list to include "Domain manager email {#}" headers @@ -135,7 +131,6 @@ def update_columns_with_domain_managers(columns, max_dm_count): for i in range(1, max_dm_count + 1): columns.append(f"Domain manager email {i}") - def write_csv( writer, columns, @@ -148,7 +143,7 @@ def write_csv( Receives params from the parent methods and outputs a CSV with fltered and sorted domains. Works with write_header as longas the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_growth_to_csv calls write_body twice + should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ all_domain_infos = get_domain_infos(filter_condition, sort_fields) @@ -158,15 +153,15 @@ def write_csv( security_emails_dict = _get_security_emails(sec_contact_ids) - # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_domain_infos, 1000) - if get_domain_managers and len(all_domain_infos) > 0: # We want to get the max amont of domain managers an # account has to set the column header dynamically max_dm_count = max(len(domain_info.domain.permissions.all()) for domain_info in all_domain_infos) update_columns_with_domain_managers(columns, max_dm_count) + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_domain_infos, 1000) + for page_num in paginator.page_range: page = paginator.page(page_num) rows = [] @@ -185,6 +180,82 @@ def write_csv( writer.writerows(rows) +def get_domain_requests(filter_condition, sort_fields): + domain_requests = ( + DomainApplication.objects.all() + .filter(**filter_condition) + .order_by(*sort_fields) + ) + + return domain_requests + +def parse_request_row(columns, request: DomainApplication): + """Given a set of columns, generate a new row from cleaned column data""" + + requested_domain_name = 'No requested domain' + + # Domain should never be none when parsing this information + if request.requested_domain is not None: + domain = request.requested_domain + requested_domain_name = domain.name + + domain = request.requested_domain # type: ignore + + if request.federal_type: + request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" + else: + request_type = request.get_organization_type_display() + + # create a dictionary of fields which can be included in output + FIELDS = { + "Requested domain": requested_domain_name, + "Status": request.get_status_display(), + "Organization type": request_type, + "Agency": request.federal_agency, + "Organization name": request.organization_name, + "City": request.city, + "State": request.state_territory, + "AO email": request.authorizing_official.email if request.authorizing_official else " ", + "Security contact email": request, + "Created at": request.created_at, + "Submission date": request.submission_date, + } + + row = [FIELDS.get(column, "") for column in columns] + return row + +def write_requests_csv( + writer, + columns, + sort_fields, + filter_condition, + should_write_header=True, +): + """ + """ + + all_requetsts = get_domain_requests(filter_condition, sort_fields) + + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_requetsts, 1000) + + for page_num in paginator.page_range: + page = paginator.page(page_num) + rows = [] + for request in page.object_list: + try: + row = parse_request_row(columns, request) + rows.append(row) + except ValueError: + # This should not happen. If it does, just skip this row. + # It indicates that DomainInformation.domain is None. + logger.error("csv_export -> Error when parsing row, domain was None") + continue + + if should_write_header: + write_header(writer, columns) + + writer.writerows(rows) def export_data_type_to_csv(csv_file): """All domains report with extra columns""" @@ -222,7 +293,6 @@ def export_data_type_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) - def export_data_full_to_csv(csv_file): """All domains report""" @@ -253,7 +323,6 @@ def export_data_full_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def export_data_federal_to_csv(csv_file): """Federal domains report""" @@ -285,18 +354,21 @@ def export_data_federal_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) - def get_default_end_date(): # Default to now() return timezone.now() +def format_start_date(start_date): + return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() + +def format_end_date(end_date): + return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() -def export_data_growth_to_csv(csv_file, start_date, end_date): +def export_data_domain_growth_to_csv(csv_file, start_date, end_date): """ Growth report: Receive start and end dates from the view, parse them. @@ -305,16 +377,9 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): the start and end dates. Specify sort params for both lists. """ - start_date_formatted = ( - timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() - ) - - end_date_formatted = ( - timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - ) - + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) writer = csv.writer(csv_file) - # define columns to include in export columns = [ "Domain name", @@ -359,3 +424,127 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): get_domain_managers=False, should_write_header=False, ) + +def get_sliced_domains(filter_condition): + """ + """ + + domains = DomainInformation.objects.all().filter(**filter_condition) + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domains.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + +def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + writer.writerow(["START DATE"]) + writer.writerow([]) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + writer.writerow(["END DATE"]) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_end_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_end_date, get_domain_managers=True, should_write_header=True) + +def export_data_requests_to_csv(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + # "domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + "submission_date__gte": start_date_formatted, + } + + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f7164663b..4d93aa54b 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -8,9 +8,32 @@ import logging logger = logging.getLogger(__name__) + +class ExportDataType(View): + def get(self, request, *args, **kwargs): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + +class ExportDataFull(View): + def get(self, request, *args, **kwargs): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + +class ExportDataFederal(View): + def get(self, request, *args, **kwargs): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response - -class ExportData(View): +class ExportDataDomainGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters # #999: not needed if we switch to django forms @@ -19,8 +42,35 @@ def get(self, request, *args, **kwargs): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use # in context to display this data in the template. - csv_export.export_data_growth_to_csv(response, start_date, end_date) + csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) return response + +class ExportDataManagedVsUnmanaged(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + csv_export.export_data_managed_vs_unamanaged_domains(response, start_date, end_date) + + return response + +class ExportDataRequests(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use + # in context to display this data in the template. + csv_export.export_data_requests_to_csv(response, start_date, end_date) + + return response \ No newline at end of file From da47cc6f7f661abba3a9dee51dc798ada775b22a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 18:46:45 -0500 Subject: [PATCH 05/27] Add charts to dashboard --- src/registrar/admin.py | 75 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 53 +++++++++++++- src/registrar/utility/csv_export.py | 27 ++++++- 3 files changed, 140 insertions(+), 15 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 9bc77b029..ca6b9bc87 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -384,39 +384,78 @@ def user_analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) + # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] - - logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") - + filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) - managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # Ready and Deleted domains + filter_ready_domains_start_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": start_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_end_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": end_date_formatted, + } + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + - # get number of ready domains, counts by org type and election office - # add to context + # Created and Submitted requests + filter_requests_start_date = { + "submission_date__lte": start_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + + filter_submitted_requests_start_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - # get number of submitted request counts by org type and election office - # add to context + filter_requests_end_date = { + "submission_date__lte": end_date_formatted, + } + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_end_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( **admin.site.each_context(request), @@ -425,10 +464,22 @@ def user_analytics(request): domain_count=models.Domain.objects.all().count(), applications_last_30_days=last_30_days_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_at_end_date=submitted_requests_at_end_date, + ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index f65aa77cf..735386d38 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -7,7 +7,10 @@ {% block content %}
-
+ +
+
+

At a glance

    @@ -19,7 +22,10 @@

    At a glance

-
+
+
+ +

Current domains

    @@ -48,6 +54,9 @@

    Current domains

+
+
+

Growth reports

@@ -119,8 +128,48 @@

Growth reports

+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
{% endblock %} diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1764536b5..ce19182b2 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -450,6 +450,31 @@ def get_sliced_domains(filter_condition): school_district, election_board] +def get_sliced_requests(filter_condition): + """ + """ + + domain_requests = DomainApplication.objects.all().filter(**filter_condition) + federal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domain_requests.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): """ """ @@ -470,7 +495,7 @@ def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) From 4b1b1386a8911718887d826950a006853ef46dbd Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 1 Mar 2024 13:44:52 -0500 Subject: [PATCH 06/27] clean up JS, init dates --- src/registrar/assets/js/get-gov-admin.js | 39 -------------------- src/registrar/templates/admin/analytics.html | 32 +++++----------- 2 files changed, 10 insertions(+), 61 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 618cc284c..2c4d4d854 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -307,45 +307,6 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, viewLink.setAttribute('title', viewLink.getAttribute('title-template').replace('selected item', elementText)); } -/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, - * attach the seleted start and end dates to a url that'll trigger the view, and finally - * redirect to that url. -*/ -(function (){ - - // Get the current date in the format YYYY-MM-DD - let currentDate = new Date().toISOString().split('T')[0]; - - // Default the value of the start date input field to the current date - let startDateInput =document.getElementById('start'); - - // Default the value of the end date input field to the current date - let endDateInput =document.getElementById('end'); - - let exportButtons = document.querySelectorAll('.exportLink'); - - if (exportButtons.length > 0) { - startDateInput.value = currentDate; - endDateInput.value = currentDate; - - exportButtons.forEach((btn) => { - btn.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = btn.dataset.exportUrl; - - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; - }); - }); - } - -})(); - /** An IIFE for admin in DjangoAdmin to listen to changes on the domain request * status select amd to show/hide the rejection reason */ diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 735386d38..6eb26307c 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -12,7 +12,7 @@

At a glance

-
+
  • User Count: {{ data.user_count }}
  • Domain Count: {{ data.domain_count }}
  • @@ -27,7 +27,7 @@

    At a glance

    Current domains

    -
    + @@ -150,18 +142,14 @@

    Growth reports

    From 32207be68a296083562c577070f98c9acd0c03cb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 1 Mar 2024 18:23:35 -0500 Subject: [PATCH 07/27] Rename, simplify, separate managed and unmanaged reports, add link to sidebar and homepage --- src/registrar/admin.py | 30 +++--- src/registrar/assets/sass/_theme/_admin.scss | 7 ++ src/registrar/config/urls.py | 31 +++--- src/registrar/templates/admin/analytics.html | 26 +++-- src/registrar/templates/admin/app_list.html | 8 +- src/registrar/templates/admin/index.html | 11 ++ src/registrar/utility/csv_export.py | 107 +++++++++++-------- src/registrar/views/admin_views.py | 32 ++++-- 8 files changed, 158 insertions(+), 94 deletions(-) create mode 100644 src/registrar/templates/admin/index.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index ca6b9bc87..ea5253ded 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -27,7 +27,6 @@ from django.utils.safestring import mark_safe from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField -from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from django.utils.translation import gettext_lazy as _ @@ -364,16 +363,18 @@ class UserContactInline(admin.StackedInline): model = models.Contact -def user_analytics(request): +def analytics(request): - end_date = datetime.datetime.today() - start_date = datetime.datetime.today() - datetime.timedelta(days=30) + thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) last_30_days_applications = models.DomainApplication.objects.filter( - created_at__gt=start_date + created_at__gt=thirty_days_ago ) - avg_approval_time = last_30_days_applications.annotate( - approval_time=F("approved_domain__created_at") - F("created_at") + last_30_days_approved_applications = models.DomainApplication.objects.filter( + created_at__gt=thirty_days_ago, status=DomainApplication.ApplicationStatus.APPROVED + ) + avg_approval_time = last_30_days_approved_applications.annotate( + approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # format the timedelta? avg_approval_time = str(avg_approval_time) @@ -416,7 +417,7 @@ def user_analytics(request): filter_deleted_domains_start_date = { "domain__state__in": [Domain.State.DELETED], - "domain__first_ready__lte": start_date_formatted, + "domain__deleted__lte": start_date_formatted, } deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) @@ -428,7 +429,7 @@ def user_analytics(request): filter_deleted_domains_end_date = { "domain__state__in": [Domain.State.DELETED], - "domain__first_ready__lte": end_date_formatted, + "domain__deleted__lte": end_date_formatted, } deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) @@ -436,7 +437,7 @@ def user_analytics(request): # Created and Submitted requests filter_requests_start_date = { - "submission_date__lte": start_date_formatted, + "created_at__lte": start_date_formatted, } requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) @@ -447,7 +448,7 @@ def user_analytics(request): submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) filter_requests_end_date = { - "submission_date__lte": end_date_formatted, + "created_at__lte": end_date_formatted, } requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) @@ -455,14 +456,15 @@ def user_analytics(request): "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( **admin.site.each_context(request), data=dict( user_count=models.User.objects.all().count(), domain_count=models.Domain.objects.all().count(), - applications_last_30_days=last_30_days_applications.count(), + last_30_days_applications=last_30_days_applications.count(), + last_30_days_approved_applications=last_30_days_approved_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, @@ -478,7 +480,7 @@ def user_analytics(request): requests_sliced_at_start_date=requests_sliced_at_start_date, submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, requests_sliced_at_end_date=requests_sliced_at_end_date, - submitted_requests_at_end_date=submitted_requests_at_end_date, + submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, ), ) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index c74daf678..fbd9ef4f7 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -320,3 +320,10 @@ input.admin-confirm-button { top: 2px; } } + +.module--custom { + a { + font-size: 13px; + font-weight: 600; + } +} diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index a9fee650e..6592ee538 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,8 +9,8 @@ from django.views.generic import RedirectView from registrar import views -from registrar.admin import user_analytics -from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType +from registrar.admin import analytics +from registrar.views.admin_views import ExportDataDomainsGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedDomains, ExportDataRequestsGrowth, ExportDataType, ExportDataUnmanagedDomains from registrar.views.application import Step from registrar.views.utility import always_404 @@ -67,24 +67,29 @@ name="export_data_federal", ), path( - "admin/analytics/export_domain_growth/", - ExportDataDomainGrowth.as_view(), - name="export_domain_growth", + "admin/analytics/export_domains_growth/", + ExportDataDomainsGrowth.as_view(), + name="export_domains_growth", ), path( - "admin/analytics/export_managed_unmanaged/", - ExportDataManagedVsUnmanaged.as_view(), - name="export_managed_unmanaged", + "admin/analytics/export_requests_growth/", + ExportDataRequestsGrowth.as_view(), + name="export_requests_growth", ), path( - "admin/analytics/export_requests/", - ExportDataRequests.as_view(), - name="export_requests", + "admin/analytics/export_managed_domains/", + ExportDataManagedDomains.as_view(), + name="export_managed_domains", + ), + path( + "admin/analytics/export_unmanaged_domains/", + ExportDataUnmanagedDomains.as_view(), + name="export_unmanaged_domains", ), path( "admin/analytics/", - admin.site.admin_view(user_analytics), - name="user_analytics", + admin.site.admin_view(analytics), + name="analytics", ), path("admin/", admin.site.urls), diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 6eb26307c..505d6dcc3 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -16,7 +16,8 @@

    At a glance

    • User Count: {{ data.user_count }}
    • Domain Count: {{ data.domain_count }}
    • -
    • Domain applications (last 30 days): {{ data.applications_last_30_days }}
    • +
    • Domain applications (last 30 days): {{ data.last_30_days_applications }}
    • +
    • Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}
    • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
    @@ -80,28 +81,35 @@

    Growth reports

    • -
    • -
    • -
    • - +
    • +
    • +
    +
    -
    -

    Growth reports

    -
    - {% comment %} - Inputs of type date suck for accessibility. - We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ - - See the commit "Review for ticket #999" - {% endcomment %} -
    -
    - - +
    +

    Growth reports

    +
    + {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ + + See the commit "Review for ticket #999" + {% endcomment %} +
    +
    + + +
    +
    + + +
    +
    +
      +
    • + +
    • +
    • + +
    • +
    • + +
    • +
    • + +
    • +
    • + +
    • +
    + +
    +
    +
    -
    - - +
    +
    -
    - -
      -
    • - -
    • -
    • - -
    • -
    • - -
    • -
    • - -
    • -
    • - -
    • -
    - -
    -
    - -
    -
    -
    -
    -
    -
    - -
    -
    - +
    +
    + +
    +
    + +
    -
    -
    -
    - -
    -
    - +
    +
    + +
    +
    + +
    -
    +
    -
    - -
    +
    {% endblock %} diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 474981dc6..bec5f3835 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -22,6 +22,7 @@ def write_header(writer, columns): """ writer.writerow(columns) + def get_domain_infos(filter_condition, sort_fields): domain_infos = ( DomainInformation.objects.select_related("domain", "authorizing_official") @@ -42,6 +43,7 @@ def get_domain_infos(filter_condition, sort_fields): ) return domain_infos_cleaned + def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" @@ -102,6 +104,7 @@ def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None row = [FIELDS.get(column, "") for column in columns] return row + def _get_security_emails(sec_contact_ids): """ Retrieve security contact emails for the given security contact IDs. @@ -123,6 +126,7 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict + def update_columns_with_domain_managers(columns, max_dm_count): """ Update the columns list to include "Domain manager email {#}" headers @@ -131,6 +135,7 @@ def update_columns_with_domain_managers(columns, max_dm_count): for i in range(1, max_dm_count + 1): columns.append(f"Domain manager email {i}") + def write_csv( writer, columns, @@ -180,19 +185,17 @@ def write_csv( writer.writerows(rows) + def get_requests(filter_condition, sort_fields): - requests = ( - DomainApplication.objects.all() - .filter(**filter_condition) - .order_by(*sort_fields) - ) + requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) return requests + def parse_request_row(columns, request: DomainApplication): """Given a set of columns, generate a new row from cleaned column data""" - requested_domain_name = 'No requested domain' + requested_domain_name = "No requested domain" # Domain should never be none when parsing this information if request.requested_domain is not None: @@ -224,6 +227,7 @@ def parse_request_row(columns, request: DomainApplication): row = [FIELDS.get(column, "") for column in columns] return row + def write_requests_csv( writer, columns, @@ -231,8 +235,7 @@ def write_requests_csv( filter_condition, should_write_header=True, ): - """ - """ + """ """ all_requetsts = get_requests(filter_condition, sort_fields) @@ -257,6 +260,7 @@ def write_requests_csv( writer.writerows(rows) + def export_data_type_to_csv(csv_file): """All domains report with extra columns""" @@ -293,6 +297,7 @@ def export_data_type_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + def export_data_full_to_csv(csv_file): """All domains report""" @@ -323,6 +328,7 @@ def export_data_full_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + def export_data_federal_to_csv(csv_file): """Federal domains report""" @@ -354,20 +360,25 @@ def export_data_federal_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) + def get_default_end_date(): # Default to now() return timezone.now() + def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() + def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + def export_data_domain_growth_to_csv(csv_file, start_date, end_date): """ Growth report: @@ -425,15 +436,17 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): should_write_header=False, ) + def get_sliced_domains(filter_condition): - """Get fitered domains counts sliced by org type and election office. - """ + """Get fitered domains counts sliced by org type and election office.""" domains = DomainInformation.objects.all().filter(**filter_condition) domains_count = domains.count() federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() - state_or_territory = domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + state_or_territory = domains.filter( + organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY + ).count() tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() @@ -441,26 +454,30 @@ def get_sliced_domains(filter_condition): school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() election_board = domains.filter(is_election_board=True).count() - return [domains_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board] + return [ + domains_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + def get_sliced_requests(filter_condition): - """Get fitered requests counts sliced by org type and election office. - """ + """Get fitered requests counts sliced by org type and election office.""" requests = DomainApplication.objects.all().filter(**filter_condition) requests_count = requests.count() federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() - state_or_territory = requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + state_or_territory = requests.filter( + organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY + ).count() tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() @@ -468,20 +485,22 @@ def get_sliced_requests(filter_condition): school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() election_board = requests.filter(is_election_board=True).count() - return [requests_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board] + return [ + requests_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get domains have domain managers for two different dates. - """ + """Get domains have domain managers for two different dates.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -501,11 +520,31 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"]) - writer.writerow(["Total", "Federal", "Interstate", "State or territory", "Tribal", "County", "City", "Special district", "School district", "Election office"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - write_csv(writer, columns, sort_fields, filter_managed_domains_start_date, get_domain_managers=True, should_write_header=True) + write_csv( + writer, + columns, + sort_fields, + filter_managed_domains_start_date, + get_domain_managers=True, + should_write_header=True, + ) writer.writerow([]) filter_managed_domains_end_date = { @@ -515,15 +554,35 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow(["Total", "Federal", "Interstate", "State or territory", "Tribal", "County", "City", "Special district", "School district", "Election office"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_csv(writer, columns, sort_fields, filter_managed_domains_end_date, get_domain_managers=True, should_write_header=True) + write_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """ Get domains that do not have domain managers for two different dates. - """ + """Get domains that do not have domain managers for two different dates.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -535,7 +594,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): sort_fields = [ "domain__name", ] - + filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, @@ -543,29 +602,69 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) - writer.writerow(["Total", "Federal", "Interstate", "State or territory", "Tribal", "County", "City", "Special district", "School district", "Election office"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - write_csv(writer, columns, sort_fields, filter_unmanaged_domains_start_date, get_domain_managers=True, should_write_header=True) + write_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_start_date, + get_domain_managers=True, + should_write_header=True, + ) writer.writerow([]) - + filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - + writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) - writer.writerow(["Total", "Federal", "Interstate", "State or territory", "Tribal", "County", "City", "Special district", "School district", "Election office"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_csv(writer, columns, sort_fields, filter_unmanaged_domains_end_date, get_domain_managers=True, should_write_header=True) + write_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + def export_data_requests_growth_to_csv(csv_file, start_date, end_date): - """ - """ + """ """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 2e21c2161..c3769ad03 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -8,7 +8,8 @@ import logging logger = logging.getLogger(__name__) - + + class ExportDataType(View): def get(self, request, *args, **kwargs): # match the CSV example with all the fields @@ -16,7 +17,8 @@ def get(self, request, *args, **kwargs): response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' csv_export.export_data_type_to_csv(response) return response - + + class ExportDataFull(View): def get(self, request, *args, **kwargs): # Smaller export based on 1 @@ -24,7 +26,8 @@ def get(self, request, *args, **kwargs): response["Content-Disposition"] = 'attachment; filename="current-full.csv"' csv_export.export_data_full_to_csv(response) return response - + + class ExportDataFederal(View): def get(self, request, *args, **kwargs): # Federal only @@ -33,6 +36,7 @@ def get(self, request, *args, **kwargs): csv_export.export_data_federal_to_csv(response) return response + class ExportDataDomainsGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters @@ -47,7 +51,8 @@ def get(self, request, *args, **kwargs): csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) return response - + + class ExportDataRequestsGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters @@ -62,7 +67,8 @@ def get(self, request, *args, **kwargs): csv_export.export_data_requests_growth_to_csv(response, start_date, end_date) return response - + + class ExportDataManagedDomains(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters @@ -70,11 +76,14 @@ def get(self, request, *args, **kwargs): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + response["Content-Disposition"] = ( + f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + ) csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) return response + class ExportDataUnmanagedDomains(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters @@ -82,7 +91,9 @@ def get(self, request, *args, **kwargs): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + response["Content-Disposition"] = ( + f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + ) csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From c89d76fb47fb9da7f40d0a29669a2d13020d7cf2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 4 Mar 2024 17:24:53 -0500 Subject: [PATCH 10/27] WIP unit testing --- src/registrar/admin.py | 49 ++- src/registrar/assets/sass/_theme/_admin.scss | 3 + src/registrar/models/domain.py | 2 +- src/registrar/templates/admin/analytics.html | 17 +- src/registrar/templates/admin/app_list.html | 2 +- src/registrar/tests/data/mocks.py | 232 ++++++++++++ src/registrar/tests/test_admin_views.py | 4 +- src/registrar/tests/test_reports.py | 370 ++++++++++++------- src/registrar/utility/csv_export.py | 109 +++--- 9 files changed, 547 insertions(+), 241 deletions(-) create mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 262cebd18..5bf41777b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -374,8 +374,8 @@ def analytics(request): avg_approval_time = last_30_days_approved_applications.annotate( approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) + # Format the timedelta to display only days + avg_approval_time = f"{avg_approval_time.days} days" start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -383,75 +383,69 @@ def analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) - # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - # Ready and Deleted domains filter_ready_domains_start_date = { "domain__state__in": [Domain.State.READY], "domain__first_ready__lte": start_date_formatted, } - ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) - - filter_deleted_domains_start_date = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": start_date_formatted, - } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - filter_ready_domains_end_date = { "domain__state__in": [Domain.State.READY], "domain__first_ready__lte": end_date_formatted, } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + filter_deleted_domains_start_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__deleted__lte": start_date_formatted, + } filter_deleted_domains_end_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": start_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - - filter_requests_end_date = { - "created_at__lte": end_date_formatted, - } - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) - filter_submitted_requests_end_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( @@ -459,6 +453,7 @@ def analytics(request): data=dict( user_count=models.User.objects.all().count(), domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(), last_30_days_applications=last_30_days_applications.count(), last_30_days_approved_applications=last_30_days_approved_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, @@ -1096,7 +1091,7 @@ def custom_election_board(self, obj): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1448,7 +1443,7 @@ def state_territory(self, obj): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] + readonly_fields = ["state", "expiration_date", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index dad88b6a4..29d0e3b2a 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -319,6 +319,9 @@ input.admin-confirm-button { .usa-icon { top: 2px; } + a.button:active, a.button:focus { + text-decoration: none; + } } .module--custom { diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..3b18ac8b6 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ def __str__(self) -> str: first_ready = DateField( null=True, - editable=False, + editable=True, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 29faffd3b..380922845 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -16,6 +16,7 @@

    At a glance

    • User Count: {{ data.user_count }}
    • Domain Count: {{ data.domain_count }}
    • +
    • Domains in READY state: {{ data.ready_domain_count }}
    • Domain applications (last 30 days): {{ data.last_30_days_applications }}
    • Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}
    • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
    • @@ -63,8 +64,6 @@

      Growth reports

      {% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ - See the commit "Review for ticket #999" {% endcomment %}
      @@ -107,7 +106,7 @@

      Growth reports

    • -
    -
    - -
    +
    + +
    diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index c96f29a31..4ee2befef 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -71,5 +71,5 @@

    Analytics

    - Dashboard + Dashboard
    \ No newline at end of file diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py new file mode 100644 index 000000000..e6dccb14f --- /dev/null +++ b/src/registrar/tests/data/mocks.py @@ -0,0 +1,232 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation +from registrar.models.domain import Domain +from registrar.models.user_domain_role import UserDomainRole +from registrar.models.public_contact import PublicContact +from registrar.models.user import User +from datetime import date, datetime, timedelta +from django.utils import timezone +from registrar.tests.common import MockEppLib + +class MockDb(MockEppLib): + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + # Test for more than 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + # Test for just 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + # self.domain_request_1, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # requested_domain=self.domain_1.name, + # organization_type="federal", + # federal_agency="World War I Centennial Commission", + # federal_type="executive", + # is_election_board=True + # ) + # self.domain_request_2, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_2, + # organization_type="interstate", + # is_election_board=True + # ) + # self.domain_request_3, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_3, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_4, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_4, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_5, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_5, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_6, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_6, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_7, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_7, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_8, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_8, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_9, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_9, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_10, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_10, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + + def tearDown(self): + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index e55175db9..cc4b3f1c7 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -3,7 +3,7 @@ from registrar.tests.common import create_superuser -class TestViews(TestCase): +class TestAdminViews(TestCase): def setUp(self): self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -26,7 +26,7 @@ def test_export_data_view(self): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index c00c2b221..43efb3128 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -8,9 +8,12 @@ from registrar.models.user import User from django.contrib.auth import get_user_model from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.common import MockEppLib +from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - write_csv, + format_end_date, + format_start_date, + get_sliced_domains, + write_domains_csv, get_default_start_date, get_default_end_date, ) @@ -231,136 +234,11 @@ def test_load_full_report(self): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockEppLib): +class ExportDataTest(MockDb): def setUp(self): super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() super().tearDown() def test_export_domains_to_writer_security_emails(self): @@ -403,7 +281,7 @@ def test_export_domains_to_writer_security_emails(self): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) @@ -427,7 +305,7 @@ def test_export_domains_to_writer_security_emails(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_csv(self): + def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" @@ -462,7 +340,7 @@ def test_write_csv(self): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -486,7 +364,7 @@ def test_write_csv(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_additional(self): + def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" with less_console_noise(): # Create a CSV file in memory @@ -512,7 +390,7 @@ def test_write_body_additional(self): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -535,7 +413,7 @@ def test_write_body_additional(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_with_date_filter_pulls_domains_in_range(self): + def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): """Test that domains that are 1. READY and their first_ready dates are in range 2. DELETED and their deleted dates are in range @@ -546,7 +424,7 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. - TODO: Simplify is created_at is not needed for the report.""" + TODO: Simplify if created_at is not needed for the report.""" with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -591,7 +469,7 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -599,7 +477,7 @@ def test_write_body_with_date_filter_pulls_domains_in_range(self): get_domain_managers=False, should_write_header=True, ) - write_csv( + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -664,7 +542,7 @@ def test_export_domains_to_writer_domain_managers(self): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True ) @@ -677,11 +555,11 @@ def test_export_domains_to_writer_domain_managers(self): expected_content = ( "Domain name,Status,Expiration date,Domain type,Agency," "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager email 1,Domain manager email 2,\n" + "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,info@example.com\n" + ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" ) # Normalize line endings and remove commas, @@ -690,6 +568,210 @@ def test_export_domains_to_writer_domain_managers(self): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + def test_export_data_managed_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + # Call the export functions + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,1\n" + "\n" + "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" + "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_export_data_unmanaged_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # Call the export functions + writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=False, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): + """Test that requests that are + 1. SUBMITTED and their submission_date are in range + are pulled when the growth report conditions are applied to export_requests_to_writed. + Test that requests are sorted by requested domain name. + """ + + pass class HelperFunctions(TestCase): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -704,3 +786,11 @@ def test_get_default_end_date(self): expected_date = timezone.now() actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) + + def get_sliced_domains(self): + """Should get fitered domains counts sliced by org type and election office.""" + pass + + def test_get_sliced_requests(self): + """Should get fitered requests counts sliced by org type and election office.""" + pass \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index bec5f3835..cbdbfddb3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -25,9 +25,10 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") + DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) + .distinct() ) # Do a mass concat of the first and last name fields for authorizing_official. @@ -44,7 +45,7 @@ def get_domain_infos(filter_condition, sort_fields): return domain_infos_cleaned -def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): +def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" # Domain should never be none when parsing this information @@ -136,7 +137,7 @@ def update_columns_with_domain_managers(columns, max_dm_count): columns.append(f"Domain manager email {i}") -def write_csv( +def write_domains_csv( writer, columns, sort_fields, @@ -145,8 +146,8 @@ def write_csv( should_write_header=True, ): """ - Receives params from the parent methods and outputs a CSV with fltered and sorted domains. - Works with write_header as longas the same writer object is passed. + Receives params from the parent methods and outputs a CSV with filtered and sorted domains. + Works with write_header as long as the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ @@ -172,7 +173,7 @@ def write_csv( rows = [] for domain_info in page.object_list: try: - row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers) + row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers) rows.append(row) except ValueError: # This should not happen. If it does, just skip this row. @@ -188,7 +189,6 @@ def write_csv( def get_requests(filter_condition, sort_fields): requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) - return requests @@ -235,7 +235,8 @@ def write_requests_csv( filter_condition, should_write_header=True, ): - """ """ + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" all_requetsts = get_requests(filter_condition, sort_fields) @@ -295,7 +296,7 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) def export_data_full_to_csv(csv_file): @@ -326,7 +327,7 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def export_data_federal_to_csv(csv_file): @@ -358,7 +359,7 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def get_default_start_date(): @@ -426,8 +427,8 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - write_csv( + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -440,19 +441,19 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): def get_sliced_domains(filter_condition): """Get fitered domains counts sliced by org type and election office.""" - domains = DomainInformation.objects.all().filter(**filter_condition) + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() domains_count = domains.count() - federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() state_or_territory = domains.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = domains.filter(is_election_board=True).count() + ).distinct().count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = domains.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -471,19 +472,19 @@ def get_sliced_domains(filter_condition): def get_sliced_requests(filter_condition): """Get fitered requests counts sliced by org type and election office.""" - requests = DomainApplication.objects.all().filter(**filter_condition) + requests = DomainApplication.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() - interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = requests.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = requests.filter(is_election_board=True).count() + ).distinct().count() + tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = requests.filter(is_election_board=True).distinct().count() return [ requests_count, @@ -500,7 +501,8 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get domains have domain managers for two different dates.""" + """Get counts for domains that have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -512,14 +514,13 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): sort_fields = [ "domain__name", ] - filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"]) + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( [ "Total", @@ -537,16 +538,6 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_managed_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, @@ -571,7 +562,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -582,7 +573,8 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get domains that do not have domain managers for two different dates.""" + """Get counts for domains that do not have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -619,16 +611,6 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, @@ -653,18 +635,23 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_unmanaged_domains_end_date, - get_domain_managers=True, + get_domain_managers=False, should_write_header=True, ) def export_data_requests_growth_to_csv(csv_file, start_date, end_date): - """ """ + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_requests_body SUBMITTED requests that are created between + the start and end dates. Specify sort params. + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -676,7 +663,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date): "Submission date", ] sort_fields = [ - # "domain__name", + "requested_domain__name", ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, From ff6149a1c42ff89ee9a5b7e8ef6a3010bb97d4df Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 12:51:22 -0500 Subject: [PATCH 11/27] Unit tests plus revert hacks to add data --- src/registrar/admin.py | 4 +- src/registrar/models/domain.py | 2 +- src/registrar/signals.py | 2 - src/registrar/tests/data/mocks.py | 84 ++++------------------ src/registrar/tests/test_reports.py | 106 ++++++++++++++++++++++++++-- src/registrar/utility/csv_export.py | 8 +-- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5bf41777b..78f85f0f9 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1091,7 +1091,7 @@ def custom_election_board(self, obj): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1443,7 +1443,7 @@ def state_territory(self, obj): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "deleted"] + readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 3b18ac8b6..449c4c4bb 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ def __str__(self) -> str: first_ready = DateField( null=True, - editable=True, + editable=False, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 74dc8a063..4e7768ef4 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,7 +27,6 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") - logger.info(f"in handle_profile first {instance}") is_new_user = kwargs.get("created", False) @@ -37,7 +36,6 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact - logger.info(f"inside no matching contacts for first {first_name} last {last_name} email {email}") Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py index e6dccb14f..25f56f247 100644 --- a/src/registrar/tests/data/mocks.py +++ b/src/registrar/tests/data/mocks.py @@ -1,5 +1,6 @@ from django.test import TestCase from django.contrib.auth import get_user_model +from api.tests.common import less_console_noise from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain @@ -8,7 +9,7 @@ from registrar.models.user import User from datetime import date, datetime, timedelta from django.utils import timezone -from registrar.tests.common import MockEppLib +from registrar.tests.common import MockEppLib, completed_application class MockDb(MockEppLib): def setUp(self): @@ -152,81 +153,22 @@ def setUp(self): user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER ) - # self.domain_request_1, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # requested_domain=self.domain_1.name, - # organization_type="federal", - # federal_agency="World War I Centennial Commission", - # federal_type="executive", - # is_election_board=True - # ) - # self.domain_request_2, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_2, - # organization_type="interstate", - # is_election_board=True - # ) - # self.domain_request_3, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_3, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_4, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_4, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_5, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_5, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_6, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_6, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_7, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_7, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_8, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_8, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_9, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_9, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_10, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_10, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() def tearDown(self): PublicContact.objects.all().delete() Domain.objects.all().delete() DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() User.objects.all().delete() UserDomainRole.objects.all().delete() super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 43efb3128..cc7cc7991 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -2,6 +2,7 @@ import io from django.test import Client, RequestFactory, TestCase from io import StringIO +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain from registrar.models.public_contact import PublicContact @@ -13,9 +14,11 @@ format_end_date, format_start_date, get_sliced_domains, + get_sliced_requests, write_domains_csv, get_default_start_date, get_default_end_date, + write_requests_csv, ) from django.core.management import call_command @@ -27,7 +30,7 @@ from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import less_console_noise +from .common import completed_application, less_console_noise class CsvReportsTest(TestCase): @@ -771,9 +774,58 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): Test that requests are sorted by requested domain name. """ - pass + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + # We use timezone.make_aware to sync to server time a datetime object with the current date + # (using date.today()) and a specific time (using datetime.min.time()). + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + # Define columns, sort fields, and filter condition + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + "requested_domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + "submission_date__gte": start_date, + } + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name + # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name + expected_content = ( + "Requested domain,Organization type,Submission date\n" + "city3.gov,Federal - Executive,2024-03-05\n" + "city4.gov,Federal - Executive,2024-03-05\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() -class HelperFunctions(TestCase): + self.assertEqual(csv_content, expected_content) + +class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" def test_get_default_start_date(self): @@ -787,10 +839,52 @@ def test_get_default_end_date(self): actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) - def get_sliced_domains(self): + def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - pass + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + + expected_content = ( + [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] + ) + + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - pass \ No newline at end of file + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + } + submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + + print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') + + expected_content = ( + [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + ) + + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index cbdbfddb3..e09258022 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -188,7 +188,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) + requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -197,12 +197,8 @@ def parse_request_row(columns, request: DomainApplication): requested_domain_name = "No requested domain" - # Domain should never be none when parsing this information if request.requested_domain is not None: - domain = request.requested_domain - requested_domain_name = domain.name - - domain = request.requested_domain # type: ignore + requested_domain_name = request.requested_domain.name if request.federal_type: request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" From 091e4c900e9853fe2600af2e0fb72e3d1a30ae80 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:28:12 -0500 Subject: [PATCH 12/27] Clean up common mock class and how it's inherited --- src/registrar/tests/common.py | 245 ++++++++++++++++++++++++---- src/registrar/tests/data/mocks.py | 174 -------------------- src/registrar/tests/test_reports.py | 166 +++++-------------- src/registrar/utility/csv_export.py | 4 +- 4 files changed, 251 insertions(+), 338 deletions(-) delete mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index ee1ab8b68..9666d135d 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -13,6 +13,8 @@ from django.conf import settings from django.contrib.auth import get_user_model, login from django.utils.timezone import make_aware +from datetime import date, datetime, timedelta +from django.utils import timezone from registrar.models import ( Contact, @@ -35,6 +37,7 @@ ErrorCode, responses, ) +from registrar.models.user_domain_role import UserDomainRole from registrar.models.utility.contact_error import ContactError, ContactErrorCodes @@ -470,6 +473,176 @@ def create_full_dummy_domain_object( application.alternative_domains.add(alt) return application + +class MockDb(TestCase): + """Hardcoded mocks make test case assertions sraightforward.""" + + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + self.end_date = current_date + timedelta(days=2) + self.start_date = current_date - timedelta(days=2) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() + + def tearDown(self): + super().tearDown() + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() def mock_user(): @@ -645,7 +818,7 @@ def dummyInfoContactResultData( self, id, email, - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), pw="thisisnotapassword", ): fake = info.InfoContactResultData( @@ -683,82 +856,82 @@ def dummyInfoContactResultData( mockDataInfoDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meoward.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainAndIPAddress = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoDomainNotSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.subdomainwoip.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataExtensionDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 11, 15), + ex_date=date(2023, 11, 15), ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "123", "123@mail.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithContacts = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -783,7 +956,7 @@ def dummyInfoContactResultData( InfoDomainWithDefaultSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultSec", @@ -798,11 +971,11 @@ def dummyInfoContactResultData( ) mockVerisignDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "defaultVeri", "registrar@dotgov.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "defaultVeri", "registrar@dotgov.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithVerisignSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultVeri", @@ -818,7 +991,7 @@ def dummyInfoContactResultData( InfoDomainWithDefaultTechnicalContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultTech", @@ -843,14 +1016,14 @@ def dummyInfoContactResultData( infoDomainNoContact = fakedEppObject( "security", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["fake.host.com"], ) infoDomainThreeHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.my-nameserver-1.com", @@ -861,43 +1034,43 @@ def dummyInfoContactResultData( infoDomainNoHost = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[], ) infoDomainTwoHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["ns1.my-nameserver-1.com", "ns1.my-nameserver-2.com"], ) mockDataInfoHosts = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="1.2.3.4"), common.Ip(addr="2.3.4.5")], ) mockDataInfoHosts1IP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoHostsNotSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 26, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 26, 19, 45, 35)), addrs=[], ) mockDataInfoHostsSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 27, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 27, 19, 45, 35)), addrs=[], ) - mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35))) + mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35))) addDsData1 = { "keyTag": 1234, "alg": 3, @@ -929,7 +1102,7 @@ def dummyInfoContactResultData( infoDomainHasIP = fakedEppObject( "nameserverwithip.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -954,7 +1127,7 @@ def dummyInfoContactResultData( justNameserver = fakedEppObject( "justnameserver.com", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -977,7 +1150,7 @@ def dummyInfoContactResultData( infoDomainCheckHostIPCombo = fakedEppObject( "nameserversubdomain.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.nameserversubdomain.gov", @@ -987,27 +1160,27 @@ def dummyInfoContactResultData( mockRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockButtonRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2025, 5, 25), + ex_date=date(2025, 5, 25), ) mockDnsNeededRenewedDomainExpDate = fakedEppObject( "fakeneeded.gov", - ex_date=datetime.date(2023, 2, 15), + ex_date=date(2023, 2, 15), ) mockMaximumRenewedDomainExpDate = fakedEppObject( "fakemaximum.gov", - ex_date=datetime.date(2024, 12, 31), + ex_date=date(2024, 12, 31), ) mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", - ex_date=datetime.date(2024, 11, 15), + ex_date=date(2024, 11, 15), ) def _mockDomainName(self, _name, _avail=False): diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py deleted file mode 100644 index 25f56f247..000000000 --- a/src/registrar/tests/data/mocks.py +++ /dev/null @@ -1,174 +0,0 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model -from api.tests.common import less_console_noise -from registrar.models.domain_application import DomainApplication -from registrar.models.domain_information import DomainInformation -from registrar.models.domain import Domain -from registrar.models.user_domain_role import UserDomainRole -from registrar.models.public_contact import PublicContact -from registrar.models.user import User -from datetime import date, datetime, timedelta -from django.utils import timezone -from registrar.tests.common import MockEppLib, completed_application - -class MockDb(MockEppLib): - def setUp(self): - super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - is_election_board=True - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - lebowski_user = get_user_model().objects.create( - username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) - - with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") - self.domain_request_3.submit() - self.domain_request_3.save() - self.domain_request_4.submit() - self.domain_request_4.save() - - def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - DomainApplication.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() - super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cc7cc7991..e6230fadb 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,14 +5,9 @@ from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain -from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model -from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - format_end_date, - format_start_date, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -30,60 +25,17 @@ from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import completed_application, less_console_noise +from .common import MockDb, MockEppLib, less_console_noise -class CsvReportsTest(TestCase): +class CsvReportsTest(MockDb): """Tests to determine if we are uploading our reports correctly""" def setUp(self): """Create fake domain data""" + super().setUp() self.client = Client(HTTP_HOST="localhost:8080") self.factory = RequestFactory() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - def tearDown(self): - """Delete all faked data""" - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - super().tearDown() @boto3_mocking.patching def test_generate_federal_report(self): @@ -94,6 +46,7 @@ def test_generate_federal_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -114,6 +67,7 @@ def test_generate_full_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -172,6 +126,7 @@ def side_effect(Bucket, Key): @boto3_mocking.patching def test_load_federal_report(self): """Tests the get_current_federal api endpoint""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -205,6 +160,7 @@ def test_load_federal_report(self): @boto3_mocking.patching def test_load_full_report(self): """Tests the current-federal api link""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -237,7 +193,7 @@ def test_load_full_report(self): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockDb): +class ExportDataTest(MockDb, MockEppLib): def setUp(self): super().setUp() @@ -247,6 +203,7 @@ def tearDown(self): def test_export_domains_to_writer_security_emails(self): """Test that export_domains_to_writer returns the expected security email""" + with less_console_noise(): # Add security email information self.domain_1.name = "defaultsecurity.gov" @@ -312,6 +269,7 @@ def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -369,6 +327,7 @@ def test_write_domains_csv(self): def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -428,15 +387,11 @@ def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): which are hard to mock. TODO: Simplify if created_at is not needed for the report.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) - # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -460,15 +415,15 @@ def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): "domain__state__in": [ Domain.State.READY, ], - "domain__first_ready__lte": end_date, - "domain__first_ready__gte": start_date, + "domain__first_ready__lte": self.end_date, + "domain__first_ready__gte": self.start_date, } filter_conditions_for_deleted_domains = { "domain__state__in": [ Domain.State.DELETED, ], - "domain__deleted__lte": end_date, - "domain__deleted__gte": start_date, + "domain__deleted__lte": self.end_date, + "domain__deleted__gte": self.start_date, } # Call the export functions @@ -515,13 +470,13 @@ def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): def test_export_domains_to_writer_domain_managers(self): """Test that export_domains_to_writer returns the - expected domain managers""" + expected domain managers.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition - columns = [ "Domain name", "Status", @@ -572,13 +527,13 @@ def test_export_domains_to_writer_domain_managers(self): self.assertEqual(csv_content, expected_content) def test_export_data_managed_domains_to_csv(self): - """""" + """Test get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -589,7 +544,7 @@ def test_export_data_managed_domains_to_csv(self): ] filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) # Call the export functions @@ -610,13 +565,11 @@ def test_export_data_managed_domains_to_csv(self): ) writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -634,7 +587,6 @@ def test_export_data_managed_domains_to_csv(self): ) writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -647,9 +599,7 @@ def test_export_data_managed_domains_to_csv(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" @@ -663,20 +613,22 @@ def test_export_data_managed_domains_to_csv(self): "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): - """""" + """Test get counts for domains that do not have domain managers for two different dates, + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -687,7 +639,7 @@ def test_export_data_unmanaged_domains_to_csv(self): ] filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) # Call the export functions @@ -708,13 +660,11 @@ def test_export_data_unmanaged_domains_to_csv(self): ) writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -732,7 +682,6 @@ def test_export_data_unmanaged_domains_to_csv(self): ) writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -745,9 +694,7 @@ def test_export_data_unmanaged_domains_to_csv(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" @@ -761,10 +708,12 @@ def test_export_data_unmanaged_domains_to_csv(self): "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -778,17 +727,6 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - # Define columns, sort fields, and filter condition columns = [ "Requested domain", @@ -800,16 +738,14 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, - "submission_date__gte": start_date, + "submission_date__lte": self.end_date, + "submission_date__gte": self.start_date, } write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) # Reset the CSV file's position to the beginning csv_file.seek(0) - # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( @@ -817,12 +753,12 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -841,50 +777,28 @@ def test_get_default_end_date(self): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] ) - self.assertEqual(managed_domains_sliced_at_end_date, expected_content) - - def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, + "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - - print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') - expected_content = ( [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1bdd3fd82..22467bf6b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -498,7 +498,7 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that have domain managers for two different dates, - get list of domains at end_date.""" + get list of managed domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -570,7 +570,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that do not have domain managers for two different dates, - get list of domains at end_date.""" + get list of unmanaged domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) From 2081f5c56483cf9dadd3f5f71d06204c3055d184 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:41:34 -0500 Subject: [PATCH 13/27] lint --- src/registrar/admin.py | 2 +- src/registrar/tests/common.py | 47 ++++++++++++++----------- src/registrar/tests/test_reports.py | 53 +++++++++++++++-------------- src/registrar/utility/csv_export.py | 46 +++++++++++++++++-------- 4 files changed, 86 insertions(+), 62 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 78f85f0f9..3d6d87367 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -424,7 +424,7 @@ def analytics(request): "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 9666d135d..e6e642918 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -1,4 +1,3 @@ -import datetime import os import logging @@ -473,7 +472,8 @@ def create_full_dummy_domain_object( application.alternative_domains.add(alt) return application - + + class MockDb(TestCase): """Hardcoded mocks make test case assertions sraightforward.""" @@ -535,69 +535,66 @@ def setUp(self): organization_type="federal", federal_agency="World War I Centennial Commission", federal_type="executive", - is_election_board=True + is_election_board=True, ) self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True + creator=self.user, domain=self.domain_2, organization_type="interstate", is_election_board=True ) self.domain_information_3, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_3, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_4, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_4, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_5, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_5, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_6, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_6, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_7, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_7, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_8, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_8, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_9, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_9, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_10, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_10, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) meoward_user = get_user_model().objects.create( @@ -625,11 +622,21 @@ def setUp(self): ) with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_1 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov" + ) + self.domain_request_2 = completed_application( + status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov" + ) + self.domain_request_3 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov" + ) + self.domain_request_4 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov" + ) + self.domain_request_5 = completed_application( + status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov" + ) self.domain_request_3.submit() self.domain_request_3.save() self.domain_request_4.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 55f5c9108..03f792825 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -43,7 +43,7 @@ def test_generate_federal_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -64,7 +64,7 @@ def test_generate_full_report(self): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -525,7 +525,7 @@ def test_export_domains_to_writer_domain_managers(self): def test_export_data_managed_domains_to_csv(self): """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" + get list of managed domains at end_date.""" with less_console_noise(): # Create a CSV file in memory @@ -596,32 +596,34 @@ def test_export_data_managed_domains_to_csv(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" "1,1,0,0,0,0,0,0,0,1\n" "\n" "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -691,26 +693,28 @@ def test_export_data_unmanaged_domains_to_csv(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "UNMANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" "\n" "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -750,14 +754,15 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) + class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -774,28 +779,24 @@ def test_get_default_end_date(self): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( - [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] - ) + expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - expected_content = ( - [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] - ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file + expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 44e34164d..fdebfef77 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -128,6 +128,7 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict + def write_domains_csv( writer, columns, @@ -253,7 +254,6 @@ def write_requests_csv( logger.error("csv_export -> Error when parsing row, domain was None") continue - if should_write_header: write_header(writer, columns) writer.writerows(rows) @@ -293,7 +293,9 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): @@ -324,7 +326,9 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): @@ -356,7 +360,9 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): @@ -424,7 +430,9 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) write_domains_csv( writer, columns, @@ -442,14 +450,18 @@ def get_sliced_domains(filter_condition): domains_count = domains.count() federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() - state_or_territory = domains.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = domains.filter(is_election_board=True).distinct().count() return [ @@ -473,14 +485,18 @@ def get_sliced_requests(filter_condition): requests_count = requests.count() federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = requests.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = requests.filter(is_election_board=True).distinct().count() return [ From 05533179b67d380e9c69d39c68bc23703ef2e1de Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:51:35 -0500 Subject: [PATCH 14/27] cleanup --- src/registrar/admin.py | 3 ++- src/registrar/templates/admin/analytics.html | 1 - src/registrar/templates/admin/app_list.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3d6d87367..41391f724 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -364,6 +364,7 @@ class UserContactInline(admin.StackedInline): def analytics(request): + """View for the reports page.""" thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) @@ -377,6 +378,7 @@ def analytics(request): # Format the timedelta to display only days avg_approval_time = f"{avg_approval_time.days} days" + # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -427,7 +429,6 @@ def analytics(request): deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) - # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 380922845..72aa244cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,7 +1,6 @@ {% extends "admin/base_site.html" %} {% load static %} - {% block content_title %}

    Registrar Analytics

    {% endblock %} {% block content %} diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index 4ee2befef..dd7e27f33 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -72,4 +72,4 @@

    Analytics

    Dashboard -
    \ No newline at end of file +
    From 11f69454c14869c28a95e6a3fd7a85578200a262 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:53:32 -0500 Subject: [PATCH 15/27] cleanup --- src/registrar/views/admin_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index c3769ad03..04f98a2c4 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -77,7 +77,7 @@ def get(self, request, *args, **kwargs): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) @@ -92,7 +92,7 @@ def get(self, request, *args, **kwargs): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) From 62cf4ecb687077a2c857f53fd2b426c9e3c08b32 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:59:37 -0500 Subject: [PATCH 16/27] lint --- src/registrar/views/admin_views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04f98a2c4..04fcaa6f2 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -76,9 +76,7 @@ def get(self, request, *args, **kwargs): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) return response @@ -91,9 +89,7 @@ def get(self, request, *args, **kwargs): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From a9878a00730ca2cadf20872207d86ea74729b64d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:01:27 -0500 Subject: [PATCH 17/27] Charts js --- src/registrar/assets/js/get-gov-reports.js | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/registrar/assets/js/get-gov-reports.js diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js new file mode 100644 index 000000000..e900fabe8 --- /dev/null +++ b/src/registrar/assets/js/get-gov-reports.js @@ -0,0 +1,117 @@ +/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, + * attach the seleted start and end dates to a url that'll trigger the view, and finally + * redirect to that url. + * + * This function also sets the start and end dates to match the url params if they exist +*/ +(function () { + // Function to get URL parameter value by name + function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); + } + + // Get the current date in the format YYYY-MM-DD + let currentDate = new Date().toISOString().split('T')[0]; + + // Default the value of the start date input field to the current date + let startDateInput = document.getElementById('start'); + + // Default the value of the end date input field to the current date + let endDateInput = document.getElementById('end'); + + let exportButtons = document.querySelectorAll('.exportLink'); + + if (exportButtons.length > 0) { + // Check if start and end dates are present in the URL + let urlStartDate = getParameterByName('start_date'); + let urlEndDate = getParameterByName('end_date'); + + // Set input values based on URL parameters or current date + startDateInput.value = urlStartDate || currentDate; + endDateInput.value = urlEndDate || currentDate; + + exportButtons.forEach((btn) => { + btn.addEventListener('click', function () { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); + }); + } + +})(); + +document.addEventListener("DOMContentLoaded", function () { + createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); + createComparativeColumnChart("myChart6", "All requests", "Start Date", "End Date"); +}); + +function createComparativeColumnChart(canvasId, title, labelOne, labelTwo) { + var canvas = document.getElementById(canvasId); + var ctx = canvas.getContext("2d"); + + var listOne = JSON.parse(canvas.getAttribute('data-list-one')); + var listTwo = JSON.parse(canvas.getAttribute('data-list-two')); + + var data = { + labels: ["Total", "Federal", "Interstate", "State/Territory", "Tribal", "County", "City", "Special District", "School District", "Election Board"], + datasets: [ + { + label: labelOne, + backgroundColor: "rgba(255, 99, 132, 0.2)", + borderColor: "rgba(255, 99, 132, 1)", + borderWidth: 1, + data: listOne, + }, + { + label: labelTwo, + backgroundColor: "rgba(75, 192, 192, 0.2)", + borderColor: "rgba(75, 192, 192, 1)", + borderWidth: 1, + data: listTwo, + }, + ], + }; + + var options = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'top', + }, + title: { + display: true, + text: title + } + }, + scales: { + y: { + beginAtZero: true, + }, + }, + }; + + new Chart(ctx, { + type: "bar", + data: data, + options: options, + }); +} \ No newline at end of file From fda2c11fb1a329d1ff03e7108484ccea52947b8d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:33:55 -0500 Subject: [PATCH 18/27] Some accessibility work on charts --- src/registrar/admin.py | 2 + src/registrar/assets/js/get-gov-reports.js | 4 +- src/registrar/templates/admin/analytics.html | 54 +++++++++++++++----- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 41391f724..5b8d67983 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -470,6 +470,8 @@ def analytics(request): submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, requests_sliced_at_end_date=requests_sliced_at_end_date, submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js index e900fabe8..d10cf2dc6 100644 --- a/src/registrar/assets/js/get-gov-reports.js +++ b/src/registrar/assets/js/get-gov-reports.js @@ -55,8 +55,8 @@ })(); document.addEventListener("DOMContentLoaded", function () { - createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); - createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart1", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Unmanaged domains", "Start Date", "End Date"); createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 72aa244cf..da7f25c66 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -115,46 +115,76 @@

    Growth reports

    - + > +

    Chart: Managed domains

    +

    {{ data.managed_domains_sliced_at_end_date.0 }} managed domains for {{ data.end_date }}

    +
    - + > +

    Chart: Unanaged domains

    +

    {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

    +
    - + > +

    Chart: Deleted domains

    +

    {{ data.deleted_domains_sliced_at_end_date.0 }} deleted domains for {{ data.end_date }}

    +
    - + > +

    Chart: Ready domains

    +

    {{ data.ready_domains_sliced_at_end_date.0 }} ready domains for {{ data.end_date }}

    +
    - + > +

    Chart: Submitted requests

    +

    {{ data.submitted_requests_sliced_at_end_date.0 }} submitted requests for {{ data.end_date }}

    +
    - + > +

    Chart: All requests

    +

    {{ data.requests_sliced_at_end_date.0 }} requests for {{ data.end_date }}

    +
    From 33b47d27d17a75d8f54281302b46f36bcab3c995 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:03:10 -0800 Subject: [PATCH 19/27] handle none --- src/registrar/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5b8d67983..68f27d15c 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,7 +376,10 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time = f"{avg_approval_time.days} days" + + avg_approval_time="No approvals to use" + if avg_approval_time is not None: + avg_approval_time = f"{avg_approval_time.days} days" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") From d1bac52aa61a1279788fc9430d593a4ec2968f1b Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:08:24 -0800 Subject: [PATCH 20/27] moved code line --- src/registrar/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 68f27d15c..4b841ab12 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -377,10 +377,11 @@ def analytics(request): ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time="No approvals to use" + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" - + else: + avg_approval_time="No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From bd352443964ebf6aae03a67c5ace122df07b03c7 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:33:22 -0800 Subject: [PATCH 21/27] lint --- src/registrar/admin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 4b841ab12..59aa2ace8 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,12 +376,11 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - - + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" else: - avg_approval_time="No approvals to use" + avg_approval_time = "No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From 8147b5f81289aaaa2d3113ea33d9f617091c765c Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Mar 2024 14:44:37 -0400 Subject: [PATCH 22/27] Fix code after merge from main (DomainRequest) --- src/registrar/admin.py | 10 +++---- src/registrar/tests/common.py | 12 ++++----- src/registrar/tests/test_reports.py | 6 ++--- src/registrar/utility/csv_export.py | 42 ++++++++++++++--------------- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index d87cb6ff6..0cf419056 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -442,9 +442,9 @@ def analytics(request): thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) - last_30_days_applications = models.DomainApplication.objects.filter(created_at__gt=thirty_days_ago) - last_30_days_approved_applications = models.DomainApplication.objects.filter( - created_at__gt=thirty_days_ago, status=DomainApplication.ApplicationStatus.APPROVED + last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) + last_30_days_approved_applications = models.DomainRequest.objects.filter( + created_at__gt=thirty_days_ago, status=DomainRequest.DomainRequestStatus.APPROVED ) avg_approval_time = last_30_days_approved_applications.annotate( approval_time=F("approved_domain__created_at") - F("submission_date") @@ -516,11 +516,11 @@ def analytics(request): requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { - "status": DomainApplication.ApplicationStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": start_date_formatted, } filter_submitted_requests_end_date = { - "status": DomainApplication.ApplicationStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 633e48ee3..f1c7841d1 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -623,19 +623,19 @@ def setUp(self): with less_console_noise(): self.domain_request_1 = completed_application( - status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov" + status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" ) self.domain_request_2 = completed_application( - status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov" + status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" ) self.domain_request_3 = completed_application( - status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov" + status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" ) self.domain_request_4 = completed_application( - status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov" + status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" ) self.domain_request_5 = completed_application( - status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov" + status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" ) self.domain_request_3.submit() self.domain_request_3.save() @@ -647,7 +647,7 @@ def tearDown(self): PublicContact.objects.all().delete() Domain.objects.all().delete() DomainInformation.objects.all().delete() - DomainApplication.objects.all().delete() + DomainRequest.objects.all().delete() User.objects.all().delete() UserDomainRole.objects.all().delete() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 03f792825..475076711 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -2,7 +2,7 @@ import io from django.test import Client, RequestFactory from io import StringIO -from registrar.models.domain_application import DomainApplication +from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain from registrar.utility.csv_export import ( get_sliced_domains, @@ -738,7 +738,7 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): "requested_domain__name", ] filter_condition = { - "status": DomainApplication.ApplicationStatus.SUBMITTED, + "status": DomainRequest.RequestStatus.SUBMITTED, "submission_date__lte": self.end_date, "submission_date__gte": self.start_date, } @@ -794,7 +794,7 @@ def test_get_sliced_requests(self): with less_console_noise(): filter_condition = { - "status": DomainApplication.ApplicationStatus.SUBMITTED, + "status": DomainRequest.RequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index fdebfef77..060c39804 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,7 +2,7 @@ import logging from datetime import datetime from registrar.models.domain import Domain -from registrar.models.domain_application import DomainApplication +from registrar.models.domain_request import DomainRequest from registrar.models.domain_information import DomainInformation from django.utils import timezone from django.core.paginator import Paginator @@ -190,11 +190,11 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() + requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() return requests -def parse_request_row(columns, request: DomainApplication): +def parse_request_row(columns, request: DomainRequest): """Given a set of columns, generate a new row from cleaned column data""" requested_domain_name = "No requested domain" @@ -448,19 +448,19 @@ def get_sliced_domains(filter_condition): domains = DomainInformation.objects.all().filter(**filter_condition).distinct() domains_count = domains.count() - federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count() state_or_territory = ( - domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) - tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) election_board = domains.filter(is_election_board=True).distinct().count() @@ -481,21 +481,21 @@ def get_sliced_domains(filter_condition): def get_sliced_requests(filter_condition): """Get fitered requests counts sliced by org type and election office.""" - requests = DomainApplication.objects.all().filter(**filter_condition).distinct() + requests = DomainRequest.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() + federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = ( - requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) - tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) election_board = requests.filter(is_election_board=True).distinct().count() @@ -679,7 +679,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date): "requested_domain__name", ] filter_condition = { - "status": DomainApplication.ApplicationStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, "submission_date__gte": start_date_formatted, } From 427e110d22e253c2267eb113d50ef4bf122696b3 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Mar 2024 18:29:06 -0400 Subject: [PATCH 23/27] Refactors, most notably reduce DB trips in get_sliced methods --- src/registrar/admin.py | 123 +-------------- src/registrar/config/settings.py | 3 +- src/registrar/config/urls.py | 4 +- src/registrar/templates/admin/analytics.html | 6 +- src/registrar/templates/admin/app_list.html | 9 +- src/registrar/tests/common.py | 12 +- src/registrar/tests/test_reports.py | 155 +++---------------- src/registrar/utility/csv_export.py | 113 ++++++++------ src/registrar/views/admin_views.py | 129 +++++++++++++++ 9 files changed, 239 insertions(+), 315 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 0cf419056..574362f2a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,13 +1,12 @@ from datetime import date import logging -import datetime import copy from django import forms -from django.db.models import Avg, F, Value, CharField, Q +from django.db.models import Value, CharField, Q from django.db.models.functions import Concat, Coalesce from django.http import HttpResponseRedirect -from django.shortcuts import redirect, render +from django.shortcuts import redirect from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin @@ -17,7 +16,6 @@ from dateutil.relativedelta import relativedelta # type: ignore from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website -from registrar.utility import csv_export from registrar.utility.errors import FSMApplicationError, FSMErrorCodes from registrar.views.utility.mixins import OrderableFieldsMixin from django.contrib.admin.views.main import ORDER_VAR @@ -437,123 +435,6 @@ class UserContactInline(admin.StackedInline): model = models.Contact -def analytics(request): - """View for the reports page.""" - - thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) - - last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) - last_30_days_approved_applications = models.DomainRequest.objects.filter( - created_at__gt=thirty_days_ago, status=DomainRequest.DomainRequestStatus.APPROVED - ) - avg_approval_time = last_30_days_approved_applications.annotate( - approval_time=F("approved_domain__created_at") - F("submission_date") - ).aggregate(Avg("approval_time"))["approval_time__avg"] - # Format the timedelta to display only days - - if avg_approval_time is not None: - avg_approval_time = f"{avg_approval_time.days} days" - else: - avg_approval_time = "No approvals to use" - # The start and end dates are passed as url params - start_date = request.GET.get("start_date", "") - end_date = request.GET.get("end_date", "") - - start_date_formatted = csv_export.format_start_date(start_date) - end_date_formatted = csv_export.format_end_date(end_date) - - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date_formatted, - } - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) - - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date_formatted, - } - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date_formatted, - } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - - filter_ready_domains_start_date = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": start_date_formatted, - } - filter_ready_domains_end_date = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - } - ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) - - filter_deleted_domains_start_date = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": start_date_formatted, - } - filter_deleted_domains_end_date = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": end_date_formatted, - } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) - - filter_requests_start_date = { - "created_at__lte": start_date_formatted, - } - filter_requests_end_date = { - "created_at__lte": end_date_formatted, - } - requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) - - filter_submitted_requests_start_date = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": start_date_formatted, - } - filter_submitted_requests_end_date = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": end_date_formatted, - } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) - - context = dict( - **admin.site.each_context(request), - data=dict( - user_count=models.User.objects.all().count(), - domain_count=models.Domain.objects.all().count(), - ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(), - last_30_days_applications=last_30_days_applications.count(), - last_30_days_approved_applications=last_30_days_approved_applications.count(), - average_application_approval_time_last_30_days=avg_approval_time, - managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, - unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, - managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, - unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, - ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, - deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, - ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, - deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, - requests_sliced_at_start_date=requests_sliced_at_start_date, - submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, - requests_sliced_at_end_date=requests_sliced_at_end_date, - submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, - start_date=start_date, - end_date=end_date, - ), - ) - return render(request, "admin/analytics.html", context) - - class MyUserAdmin(BaseUserAdmin): """Custom user admin class to use our inlines.""" diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index b67c9356b..646b7298f 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -330,7 +330,8 @@ # Google analytics requires that we relax our otherwise # strict CSP by allowing scripts to run from their domain -# and inline with a nonce, as well as allowing connections back to their domain +# and inline with a nonce, as well as allowing connections back to their domain. +# Note: If needed, we can embed chart.js instead of using the CDN CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 38e490db8..663c6914c 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,7 +9,6 @@ from django.views.generic import RedirectView from registrar import views -from registrar.admin import analytics from registrar.views.admin_views import ( ExportDataDomainsGrowth, ExportDataFederal, @@ -18,6 +17,7 @@ ExportDataRequestsGrowth, ExportDataType, ExportDataUnmanagedDomains, + AnalyticsView, ) from registrar.views.domain_request import Step @@ -96,7 +96,7 @@ ), path( "admin/analytics/", - admin.site.admin_view(analytics), + AnalyticsView.as_view(), name="analytics", ), path("admin/", admin.site.urls), diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index da7f25c66..2c5963e75 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -46,7 +46,7 @@

    Current domains

    Current Federal + Current federal
@@ -87,7 +87,7 @@

Growth reports

  • @@ -132,7 +132,7 @@

    Chart: Managed domains

    data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}" data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}" > -

    Chart: Unanaged domains

    +

    Chart: Unmanaged domains

    {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

  • diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index dd7e27f33..49fb59e79 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -64,12 +64,11 @@
    {% endfor %} +
    +

    Analytics

    + Dashboard +
    {% else %}

    {% translate 'You don’t have permission to view or edit anything.' %}

    {% endif %} - -
    -

    Analytics

    - Dashboard -
    diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index f1c7841d1..48b42f47c 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -475,7 +475,7 @@ def create_full_dummy_domain_object( class MockDb(TestCase): - """Hardcoded mocks make test case assertions sraightforward.""" + """Hardcoded mocks make test case assertions straightforward.""" def setUp(self): super().setUp() @@ -622,19 +622,19 @@ def setUp(self): ) with less_console_noise(): - self.domain_request_1 = completed_application( + self.domain_request_1 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" ) - self.domain_request_2 = completed_application( + self.domain_request_2 = completed_domain_request( status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" ) - self.domain_request_3 = completed_application( + self.domain_request_3 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" ) - self.domain_request_4 = completed_application( + self.domain_request_4 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" ) - self.domain_request_5 = completed_application( + self.domain_request_5 = completed_domain_request( status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" ) self.domain_request_3.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 475076711..b91f3bd18 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,6 +5,8 @@ from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain from registrar.utility.csv_export import ( + export_data_managed_domains_to_csv, + export_data_unmanaged_domains_to_csv, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -530,68 +532,10 @@ def test_export_data_managed_domains_to_csv(self): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.start_date, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - # Call the export functions - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.end_date, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_managed_domains_end_date, - get_domain_managers=True, - should_write_header=True, + export_data_managed_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -627,68 +571,10 @@ def test_export_data_unmanaged_domains_to_csv(self): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.start_date, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) - # Call the export functions - writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.end_date, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_end_date, - get_domain_managers=False, - should_write_header=True, + export_data_unmanaged_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -696,12 +582,12 @@ def test_export_data_unmanaged_domains_to_csv(self): self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( - "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "UNMANAGED DOMAINS AT START DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" - "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "UNMANAGED DOMAINS AT END DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" @@ -729,16 +615,17 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition + # We'll skip submission date because it's dynamic and therefore + # impossible to set in expected_content columns = [ "Requested domain", "Organization type", - "Submission date", ] sort_fields = [ "requested_domain__name", ] filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, "submission_date__gte": self.start_date, } @@ -750,9 +637,9 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( - "Requested domain,Organization type,Submission date\n" - "city3.gov,Federal - Executive,2024-03-05\n" - "city4.gov,Federal - Executive,2024-03-05\n" + "Requested domain,Organization type\n" + "city3.gov,Federal - Executive\n" + "city4.gov,Federal - Executive\n" ) # Normalize line endings and remove commas, @@ -785,16 +672,22 @@ def test_get_sliced_domains(self): "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + # Test with distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True) expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + # Test without distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1] + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" with less_console_noise(): filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 060c39804..e8746eafb 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from collections import Counter import csv import logging from datetime import datetime @@ -25,7 +26,8 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") + DomainInformation.objects.select_related("domain", "authorizing_official") + .prefetch_related("domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) .distinct() @@ -190,7 +192,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() + requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -236,10 +238,10 @@ def write_requests_csv( """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. Works with write_header as long as the same writer object is passed.""" - all_requetsts = get_requests(filter_condition, sort_fields) + all_requests = get_requests(filter_condition, sort_fields) # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_requetsts, 1000) + paginator = Paginator(all_requests, 1000) for page_num in paginator.page_range: page = paginator.page(page_num) @@ -443,26 +445,37 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): ) -def get_sliced_domains(filter_condition): - """Get fitered domains counts sliced by org type and election office.""" +def get_sliced_domains(filter_condition, distinct=False): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count() - state_or_territory = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = domains.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct domain names based on filter condition + domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ domains_count, @@ -478,26 +491,34 @@ def get_sliced_domains(filter_condition): ] -def get_sliced_requests(filter_condition): - """Get fitered requests counts sliced by org type and election office.""" +def get_sliced_requests(filter_condition, distinct=False): + """Get filtered requests counts sliced by org type and election office.""" - requests = DomainRequest.objects.all().filter(**filter_condition).distinct() - requests_count = requests.count() - federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = requests.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct requests based on filter condition + requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ requests_count, @@ -531,7 +552,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( @@ -555,7 +576,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( @@ -604,7 +625,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True) writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) writer.writerow( @@ -628,7 +649,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True) writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) writer.writerow( diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04fcaa6f2..eba8423ed 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -2,6 +2,12 @@ from django.http import HttpResponse from django.views import View +from django.shortcuts import render +from django.contrib import admin +from django.db.models import Avg, F +from .. import models +import datetime +from django.utils import timezone from registrar.utility import csv_export @@ -10,6 +16,129 @@ logger = logging.getLogger(__name__) +class AnalyticsView(View): + def get(self, request): + thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) + thirty_days_ago = timezone.make_aware(thirty_days_ago) + + last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) + last_30_days_approved_applications = models.DomainRequest.objects.filter( + created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED + ) + avg_approval_time = last_30_days_approved_applications.annotate( + approval_time=F("approved_domain__created_at") - F("submission_date") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # Format the timedelta to display only days + if avg_approval_time is not None: + avg_approval_time_display = f"{avg_approval_time.days} days" + else: + avg_approval_time_display = "No approvals to use" + + # The start and end dates are passed as url params + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains( + filter_unmanaged_domains_start_date, True + ) + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True) + + filter_ready_domains_start_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + filter_ready_domains_end_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": start_date_formatted, + } + filter_deleted_domains_end_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + filter_requests_start_date = { + "created_at__lte": start_date_formatted, + } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_start_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + filter_submitted_requests_end_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + + context = dict( + # Generate a dictionary of context variables that are common across all admin templates + # (site_header, site_url, ...), + # include it in the larger context dictionary so it's available in the template rendering context. + # This ensures that the admin interface styling and behavior are consistent with other admin pages. + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(), + last_30_days_applications=last_30_days_applications.count(), + last_30_days_approved_applications=last_30_days_approved_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time_display, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, + ), + ) + return render(request, "admin/analytics.html", context) + + class ExportDataType(View): def get(self, request, *args, **kwargs): # match the CSV example with all the fields From 9ecd34593c15b439e9b9d09312cd798e6a73969d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Mar 2024 18:31:58 -0400 Subject: [PATCH 24/27] make update charts button stand out more --- src/registrar/templates/admin/analytics.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2c5963e75..e73f22ec5 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -105,7 +105,7 @@

    Growth reports

  • -