From f4ff2a1973179ff5be0a33858a42ec8eb15dd3ae Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Fri, 22 Nov 2024 16:59:26 +0000 Subject: [PATCH 1/8] PC-1465: Create method to filter results to newest EPC for each UPRN --- help_to_heat/frontdoor/views.py | 4 ++-- help_to_heat/utils.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/help_to_heat/frontdoor/views.py b/help_to_heat/frontdoor/views.py index 4728128f..228b057c 100644 --- a/help_to_heat/frontdoor/views.py +++ b/help_to_heat/frontdoor/views.py @@ -624,9 +624,9 @@ def save_post_data(self, data, session_id, page_name): try: if country != country_field_scotland: address_and_lmk_details = interface.api.epc.get_address_and_epc_lmk(building_name_or_number, postcode) - if len(address_and_lmk_details) > 0: - data[address_all_address_and_lmk_details_field] = address_and_lmk_details + most_recent_address_and_lmk_details = utils.get_most_recent_epc_per_uprn(address_and_lmk_details) + data[address_all_address_and_lmk_details_field] = most_recent_address_and_lmk_details data[address_choice_field] = address_choice_field_write_address else: data[address_choice_field] = address_choice_field_epc_api_fail diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index 88aa8bfe..0b689d1e 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -6,6 +6,7 @@ import secrets import types import uuid +import itertools from datetime import datetime import marshmallow @@ -249,3 +250,19 @@ def get_current_and_next_month_names(month_names): # this is to tell the user the first month we have no EPCs for, and the next month we expect to perform a dump def get_current_scottish_epc_cutoff_and_next_dump_month_names(month_names): return month_names[9], month_names[1] + + +def get_most_recent_epc_per_uprn(address_and_lmk_details): + most_recent_address_and_lmk_details = [] + uprn_groups = [] + address_and_lmk_details.sort(key=lambda x: x['uprn'], reverse=True) + for _, group in itertools.groupby(address_and_lmk_details, lambda x: x['uprn']): + uprn_groups.append(list(group)) + print(str(len(uprn_groups))) + for group in uprn_groups: + print(group) + item = max(group, key=lambda x: datetime.strptime(x['lodgement-date'], '%Y-%m-%d')) + print(item['lodgement-date']) + most_recent_address_and_lmk_details.append(item) + + return most_recent_address_and_lmk_details From 1ec3e6f8931d36bcb390831051ef5af4a6aa8586 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Fri, 22 Nov 2024 17:08:15 +0000 Subject: [PATCH 2/8] PC-1465: Refactor for efficiency --- help_to_heat/utils.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index 0b689d1e..62a1e832 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -254,15 +254,10 @@ def get_current_scottish_epc_cutoff_and_next_dump_month_names(month_names): def get_most_recent_epc_per_uprn(address_and_lmk_details): most_recent_address_and_lmk_details = [] - uprn_groups = [] address_and_lmk_details.sort(key=lambda x: x['uprn'], reverse=True) + for _, group in itertools.groupby(address_and_lmk_details, lambda x: x['uprn']): - uprn_groups.append(list(group)) - print(str(len(uprn_groups))) - for group in uprn_groups: - print(group) - item = max(group, key=lambda x: datetime.strptime(x['lodgement-date'], '%Y-%m-%d')) - print(item['lodgement-date']) + item = max(list(group), key=lambda x: datetime.strptime(x['lodgement-date'], '%Y-%m-%d')) most_recent_address_and_lmk_details.append(item) return most_recent_address_and_lmk_details From 4d0790b5acc6f001cebef148097ca355ca608b84 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Mon, 25 Nov 2024 11:55:10 +0000 Subject: [PATCH 3/8] PC-1465: Add test and reformat code --- .../sample_search_response.json | 94 +++++++++++++++++++ help_to_heat/utils.py | 8 +- tests/test_frontdoor.py | 37 ++++++++ 3 files changed, 135 insertions(+), 4 deletions(-) diff --git a/help_to_heat/frontdoor/mock_epc_api_data/sample_search_response.json b/help_to_heat/frontdoor/mock_epc_api_data/sample_search_response.json index 62c8ec8f..39839dec 100644 --- a/help_to_heat/frontdoor/mock_epc_api_data/sample_search_response.json +++ b/help_to_heat/frontdoor/mock_epc_api_data/sample_search_response.json @@ -281,6 +281,100 @@ "low-energy-lighting": "0", "walls-description": "Sandstone, as built, no insulation (assumed)", "hotwater-description": "Electric immersion, standard tariff" + }, + { + "low-energy-fixed-light-count": "", + "address": "22 Acacia Avenue, Upper Wellgood, Fulchester, FL23 4JA", + "uprn-source": "Address Matched", + "floor-height": "2.833", + "heating-cost-potential": "1100", + "unheated-corridor-length": "2.05", + "hot-water-cost-potential": "166", + "construction-age-band": "England and Wales: before 1900", + "potential-energy-rating": "F", + "mainheat-energy-eff": "Very Poor", + "windows-env-eff": "Average", + "lighting-energy-eff": "Very Poor", + "environment-impact-potential": "28", + "glazed-type": "double glazing, unknown install date", + "heating-cost-current": "1916", + "address3": "", + "mainheatcont-description": "Programmer and appliance thermostats", + "sheating-energy-eff": "N/A", + "property-type": "Maisonette", + "local-authority-label": "Westminster", + "fixed-lighting-outlets-count": "", + "energy-tariff": "Unknown", + "mechanical-ventilation": "natural", + "hot-water-cost-current": "365", + "county": "Greater London Authority", + "postcode": "FL23 4JA", + "solar-water-heating-flag": "N", + "constituency": "E14000639", + "co2-emissions-potential": "9.9", + "number-heated-rooms": "3", + "floor-description": "(other premises below)", + "energy-consumption-potential": "602", + "local-authority": "E09000033", + "built-form": "NO DATA!", + "number-open-fireplaces": "0", + "windows-description": "Fully double glazed", + "glazed-area": "Normal", + "inspection-date": "2010-07-20", + "mains-gas-flag": "N", + "co2-emiss-curr-per-floor-area": "81", + "address1": "22 Acacia Avenue", + "heat-loss-corridor": "unheated corridor", + "flat-storey-count": "5.0", + "constituency-label": "Cities of London and Westminster", + "roof-energy-eff": "Very Poor", + "total-floor-area": "109.021", + "building-reference-number": "8502528768", + "environment-impact-current": "33", + "co2-emissions-current": "8.8", + "roof-description": "Pitched, no insulation (assumed)", + "floor-energy-eff": "N/A", + "number-habitable-rooms": "3", + "address2": "Upper Wellgood", + "hot-water-env-eff": "Poor", + "posttown": "Fulchester", + "mainheatc-energy-eff": "Good", + "main-fuel": "electricity - this is for backwards compatibility only and should not be used", + "lighting-env-eff": "Very Poor", + "windows-energy-eff": "Average", + "floor-env-eff": "N/A", + "sheating-env-eff": "N/A", + "lighting-description": "No low energy lighting", + "roof-env-eff": "Very Poor", + "walls-energy-eff": "Very Poor", + "photo-supply": "0.0", + "lighting-cost-potential": "136", + "mainheat-env-eff": "Poor", + "multi-glaze-proportion": "100", + "main-heating-controls": "2603", + "lodgement-datetime": "2010-07-20 16:38:12", + "flat-top-storey": "Y", + "current-energy-rating": "G", + "secondheat-description": "None", + "walls-env-eff": "Very Poor", + "transaction-type": "rental (private)", + "uprn": "001234567890", + "current-energy-efficiency": "8", + "energy-consumption-current": "538", + "mainheat-description": "Room heaters, electric", + "lighting-cost-current": "121", + "lodgement-date": "2010-07-20", + "extension-count": "0", + "mainheatc-env-eff": "Good", + "lmk-key": "333333333333333333333333333333333", + "wind-turbine-count": "0", + "tenure": "rental (private)", + "floor-level": "3rd", + "potential-energy-efficiency": "35", + "hot-water-energy-eff": "Very Poor", + "low-energy-lighting": "0", + "walls-description": "Sandstone, as built, no insulation (assumed)", + "hotwater-description": "Electric immersion, standard tariff" } ] } \ No newline at end of file diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index 62a1e832..a8af6ff3 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -3,10 +3,10 @@ import functools import hashlib import inspect +import itertools import secrets import types import uuid -import itertools from datetime import datetime import marshmallow @@ -254,10 +254,10 @@ def get_current_scottish_epc_cutoff_and_next_dump_month_names(month_names): def get_most_recent_epc_per_uprn(address_and_lmk_details): most_recent_address_and_lmk_details = [] - address_and_lmk_details.sort(key=lambda x: x['uprn'], reverse=True) + address_and_lmk_details.sort(key=lambda x: x["uprn"], reverse=True) - for _, group in itertools.groupby(address_and_lmk_details, lambda x: x['uprn']): - item = max(list(group), key=lambda x: datetime.strptime(x['lodgement-date'], '%Y-%m-%d')) + for _, group in itertools.groupby(address_and_lmk_details, lambda x: x["uprn"]): + item = max(list(group), key=lambda x: datetime.strptime(x["lodgement-date"], "%Y-%m-%d")) most_recent_address_and_lmk_details.append(item) return most_recent_address_and_lmk_details diff --git a/tests/test_frontdoor.py b/tests/test_frontdoor.py index aa05b550..a48e3895 100644 --- a/tests/test_frontdoor.py +++ b/tests/test_frontdoor.py @@ -2112,6 +2112,43 @@ def test_epc_page_shows_epc_info(): assert page.has_one("p:contains('23 July 2010')") +@unittest.mock.patch("help_to_heat.frontdoor.interface.EPCApi", MockEPCApi) +def test_epc_select_only_shows_most_recent_epc_per_uprn(): + client = utils.get_client() + page = client.get("/start") + assert page.status_code == 302 + page = page.follow() + + assert page.status_code == 200 + session_id = page.path.split("/")[1] + assert uuid.UUID(session_id) + _check_page = _make_check_page(session_id) + + form = page.get_form() + form["country"] = "England" + page = form.submit().follow() + + form = page.get_form() + form["supplier"] = "Utilita" + page = form.submit().follow() + + assert page.has_text("Do you own the property?") + page = _check_page(page, "own-property", "own_property", "Yes, I own my property and live in it") + + assert page.has_text("Do you live in a park home") + page = _check_page(page, "park-home", "park_home", "No") + + form = page.get_form() + form["building_name_or_number"] = "22" + form["postcode"] = "FL23 4JA" + page = form.submit().follow() + + data = interface.api.session.get_answer(session_id, page_name="address") + assert len(data["address_and_lmk_details"]) == 2 + assert data["address_and_lmk_details"][0]["lmk-key"] != "3333333333333333333333333333333333" + assert data["address_and_lmk_details"][1]["lmk-key"] != "3333333333333333333333333333333333" + + @unittest.mock.patch("help_to_heat.frontdoor.interface.EPCApi", MockEPCApi) def test_success_page_still_shows_if_journey_cannot_reach_it(): supplier = "Utilita" From 604c181907d17acd8134bf8280c4cc5c7e57a672 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Mon, 25 Nov 2024 12:00:27 +0000 Subject: [PATCH 4/8] PC-1465: Extract translations --- help_to_heat/locale/cy/LC_MESSAGES/django.po | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/help_to_heat/locale/cy/LC_MESSAGES/django.po b/help_to_heat/locale/cy/LC_MESSAGES/django.po index 787e6cd7..87cee2c6 100644 --- a/help_to_heat/locale/cy/LC_MESSAGES/django.po +++ b/help_to_heat/locale/cy/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-11-15 17:05+0000\n" +"POT-Creation-Date: 2024-11-25 11:59+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -1585,11 +1585,11 @@ msgid "" "updated." msgstr "" "Os oes EPC newydd wedi'i rhoi i’ch eiddo chi ers 1 " -"%(scottish_epc_cutoff_month)s, efallai na fyddwn yn gallu dod o hyd iddi yn ein " -"cronfa ddata am mai dim ond bob chwarter y bydd honno’n cael ei diweddaru. " -"Os nad ydych chi’n adnabod yr EPC isod fel eich un ddiweddaraf chi, dewch yn " -"ôl i'r gwasanaeth yma ar ôl 1 %(next_scottish_dump_month)s pan ydyn ni’n disgwyl y " -"bydd y gronfa ddata wedi'i diweddaru." +"%(scottish_epc_cutoff_month)s, efallai na fyddwn yn gallu dod o hyd iddi yn " +"ein cronfa ddata am mai dim ond bob chwarter y bydd honno’n cael ei " +"diweddaru. Os nad ydych chi’n adnabod yr EPC isod fel eich un ddiweddaraf " +"chi, dewch yn ôl i'r gwasanaeth yma ar ôl 1 %(next_scottish_dump_month)s pan " +"ydyn ni’n disgwyl y bydd y gronfa ddata wedi'i diweddaru." #: help_to_heat/templates/frontdoor/epc.html:29 msgid "" @@ -2123,8 +2123,8 @@ msgstr "" "Os yw’ch eiddo wedi cael EPC ers 1 %(scottish_epc_cutoff_month)s, efallai na " "fyddwn yn gallu dod o hyd iddi yn ein cronfa ddata am mai bob chwarter y " "bydd honno’n cael ei diweddaru. Dewch yn ôl i’r gwasanaeth yma ar ôl 1 " -"%(next_scottish_dump_month)s pan ydyn ni’n disgwyl y bydd y gronfa ddata wedi'i " -"diweddaru." +"%(next_scottish_dump_month)s pan ydyn ni’n disgwyl y bydd y gronfa ddata " +"wedi'i diweddaru." #: help_to_heat/templates/frontdoor/no-epc.html:27 msgid "You may continue using this service without an EPC." From 609442d48c747de15259557aa1121e968188d319 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Mon, 25 Nov 2024 16:58:12 +0000 Subject: [PATCH 5/8] PC-1465: Refactor & add exception for EPC with no UPRN --- help_to_heat/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index a8af6ff3..ff230da8 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -254,10 +254,13 @@ def get_current_scottish_epc_cutoff_and_next_dump_month_names(month_names): def get_most_recent_epc_per_uprn(address_and_lmk_details): most_recent_address_and_lmk_details = [] - address_and_lmk_details.sort(key=lambda x: x["uprn"], reverse=True) - - for _, group in itertools.groupby(address_and_lmk_details, lambda x: x["uprn"]): - item = max(list(group), key=lambda x: datetime.strptime(x["lodgement-date"], "%Y-%m-%d")) - most_recent_address_and_lmk_details.append(item) + sorted_address_and_lmk_details = sorted(address_and_lmk_details, key=lambda x: x["uprn"], reverse=True) + + for uprn, group in itertools.groupby(sorted_address_and_lmk_details, lambda x: x["uprn"]): + if uprn == "": + most_recent_address_and_lmk_details.extend(group) + continue + latest_epc = max(group, key=lambda x: datetime.strptime(x["lodgement-date"], "%Y-%m-%d")) + most_recent_address_and_lmk_details.append(latest_epc) return most_recent_address_and_lmk_details From f552915c9d6d8bbef4a847de441233cf8815e9e8 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Mon, 25 Nov 2024 17:16:33 +0000 Subject: [PATCH 6/8] PC-1465: Update test to use standard strings --- tests/test_frontdoor.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_frontdoor.py b/tests/test_frontdoor.py index a48e3895..a27593f9 100644 --- a/tests/test_frontdoor.py +++ b/tests/test_frontdoor.py @@ -6,7 +6,11 @@ from django.utils import timezone from help_to_heat.frontdoor import interface -from help_to_heat.frontdoor.consts import country_page +from help_to_heat.frontdoor.consts import ( + address_all_address_and_lmk_details_field, + address_page, + country_page, +) from help_to_heat.frontdoor.mock_epc_api import ( MockEPCApi, MockEPCApiWithEPCC, @@ -2143,10 +2147,13 @@ def test_epc_select_only_shows_most_recent_epc_per_uprn(): form["postcode"] = "FL23 4JA" page = form.submit().follow() - data = interface.api.session.get_answer(session_id, page_name="address") - assert len(data["address_and_lmk_details"]) == 2 - assert data["address_and_lmk_details"][0]["lmk-key"] != "3333333333333333333333333333333333" - assert data["address_and_lmk_details"][1]["lmk-key"] != "3333333333333333333333333333333333" + data = interface.api.session.get_answer(session_id, page_name=address_page) + + assert page.has_one("label:contains('22 Acacia Avenue, Upper Wellgood, Fulchester, FL23 4JA')") + + assert len(data[address_all_address_and_lmk_details_field]) == 2 + assert data[address_all_address_and_lmk_details_field][0]["lmk-key"] != "3333333333333333333333333333333333" + assert data[address_all_address_and_lmk_details_field][1]["lmk-key"] != "3333333333333333333333333333333333" @unittest.mock.patch("help_to_heat.frontdoor.interface.EPCApi", MockEPCApi) From a7c73a0aeae2f225d42848900dfa652dd7332cfa Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Tue, 26 Nov 2024 09:52:29 +0000 Subject: [PATCH 7/8] PC-1465: Change test to add both addresses, and update to .get accessor --- help_to_heat/utils.py | 4 ++-- tests/test_frontdoor.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index ff230da8..affa09f9 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -254,9 +254,9 @@ def get_current_scottish_epc_cutoff_and_next_dump_month_names(month_names): def get_most_recent_epc_per_uprn(address_and_lmk_details): most_recent_address_and_lmk_details = [] - sorted_address_and_lmk_details = sorted(address_and_lmk_details, key=lambda x: x["uprn"], reverse=True) + sorted_address_and_lmk_details = sorted(address_and_lmk_details, key=lambda x: x.get("uprn", ""), reverse=True) - for uprn, group in itertools.groupby(sorted_address_and_lmk_details, lambda x: x["uprn"]): + for uprn, group in itertools.groupby(sorted_address_and_lmk_details, lambda x: x.get("uprn", "")): if uprn == "": most_recent_address_and_lmk_details.extend(group) continue diff --git a/tests/test_frontdoor.py b/tests/test_frontdoor.py index a27593f9..51739c06 100644 --- a/tests/test_frontdoor.py +++ b/tests/test_frontdoor.py @@ -2150,6 +2150,7 @@ def test_epc_select_only_shows_most_recent_epc_per_uprn(): data = interface.api.session.get_answer(session_id, page_name=address_page) assert page.has_one("label:contains('22 Acacia Avenue, Upper Wellgood, Fulchester, FL23 4JA')") + assert page.has_one("label:contains('11 Acacia Avenue, Upper Wellgood, Fulchester, FL23 4JA')") assert len(data[address_all_address_and_lmk_details_field]) == 2 assert data[address_all_address_and_lmk_details_field][0]["lmk-key"] != "3333333333333333333333333333333333" From 8e0b5297223afffa818c8c870da78e48eb039426 Mon Sep 17 00:00:00 2001 From: Glenn Clarke Date: Tue, 26 Nov 2024 14:07:38 +0000 Subject: [PATCH 8/8] PC-1465: Add a comment to explain EPC without UPRN exception --- help_to_heat/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/help_to_heat/utils.py b/help_to_heat/utils.py index affa09f9..6d522a24 100644 --- a/help_to_heat/utils.py +++ b/help_to_heat/utils.py @@ -257,6 +257,8 @@ def get_most_recent_epc_per_uprn(address_and_lmk_details): sorted_address_and_lmk_details = sorted(address_and_lmk_details, key=lambda x: x.get("uprn", ""), reverse=True) for uprn, group in itertools.groupby(sorted_address_and_lmk_details, lambda x: x.get("uprn", "")): + # Some EPCs might not have UPRNs, this is currently unobserved but technically possible. + # This is a failsafe to include them all in the output, as we are unable to filter them by UPRN if uprn == "": most_recent_address_and_lmk_details.extend(group) continue