From 0ec5b6a494734248595ffe0bb0c72894003d1481 Mon Sep 17 00:00:00 2001 From: wabinyai Date: Mon, 15 Apr 2024 23:14:38 +0300 Subject: [PATCH 1/3] Gaseous Pollutants --- src/analytics/api/utils/pollutants/pm_25.py | 1 + src/analytics/api/views/data.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/analytics/api/utils/pollutants/pm_25.py b/src/analytics/api/utils/pollutants/pm_25.py index a45053c4f4..9527caf00f 100644 --- a/src/analytics/api/utils/pollutants/pm_25.py +++ b/src/analytics/api/utils/pollutants/pm_25.py @@ -59,6 +59,7 @@ "pm2_5": ["pm2_5", "s1_pm2_5", "s2_pm2_5"], "pm10": ["pm10", "s1_pm10", "s2_pm10"], "no2": ["no2"], + "voc": ["voc"], }, "daily": { "pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"], diff --git a/src/analytics/api/views/data.py b/src/analytics/api/views/data.py index e30cd6fec9..fd667757c1 100644 --- a/src/analytics/api/views/data.py +++ b/src/analytics/api/views/data.py @@ -54,7 +54,7 @@ class DataExportResource(Resource): "airqlouds|optional:list", ) def post(self): - valid_pollutants = ["pm2_5", "pm10", "no2"] + valid_pollutants = ["pm2_5", "pm10", "no2","voc"] valid_download_types = ["csv", "json"] valid_output_formats = ["airqo-standard", "aqcsv"] valid_frequencies = ["hourly", "daily", "raw"] @@ -199,7 +199,7 @@ class DataExportV2Resource(Resource): "meta_data|optional:dict", ) def post(self): - valid_pollutants = ["pm2_5", "pm10", "no2"] + valid_pollutants = ["pm2_5", "pm10", "no2","voc"] valid_export_formats = ["csv", "json"] valid_output_formats = ["airqo-standard", "aqcsv"] valid_frequencies = ["hourly", "daily", "raw"] From fc4be8000648c619954403198a14150aa8420df3 Mon Sep 17 00:00:00 2001 From: wabinyai Date: Tue, 16 Apr 2024 23:21:49 +0300 Subject: [PATCH 2/3] database --- src/analytics/api/models/events.py | 35 +++++++++++++---- src/analytics/api/utils/pollutants/pm_25.py | 42 ++++++++++++++++++++- src/analytics/api/views/data.py | 4 +- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/analytics/api/models/events.py b/src/analytics/api/models/events.py index c72624fa05..a8cccb1836 100644 --- a/src/analytics/api/models/events.py +++ b/src/analytics/api/models/events.py @@ -36,7 +36,7 @@ class EventsModel(BasePyMongoModel): DEVICES_SUMMARY_TABLE = CONFIGURATIONS.DEVICES_SUMMARY_TABLE def __init__(self, tenant): - self.limit_mapper = {"pm2_5": 500.5, "pm10": 604.5, "no2": 2049} + self.limit_mapper = {"pm2_5": 500.5, "pm10": 604.5, "no2": 2049, "co": 50.5 ,"nh3":2500} super().__init__(tenant, collection_name="events") @classmethod @@ -95,6 +95,14 @@ def download_from_bigquery( bam_pollutant_columns.extend( ["no2 as no2_raw_value", "no2 as no2_calibrated_value"] ) + elif pollutant == "co": + bam_pollutant_columns.extend( + ["co as co_raw_value", "co as co_calibrated_value"] + ) + elif pollutant == "nh3": + bam_pollutant_columns.extend( + ["nh3 as co_raw_value", "nh3 as co_calibrated_value"] + ) pollutants_query = ( f" SELECT {', '.join(map(str, set(pollutant_columns)))} ," @@ -315,7 +323,14 @@ def data_export_query( bam_pollutant_columns.extend( ["no2 as no2_raw_value", "no2 as no2_calibrated_value"] ) - + elif pollutant == "co": + bam_pollutant_columns.extend( + ["co as co_raw_value", "co as co_calibrated_value"] + ) + elif pollutant == "nh3": + bam_pollutant_columns.extend( + ["nh3 as nh3_raw_value", "nh3 as nh3_calibrated_value"] + ) pollutants_query = ( f" SELECT {', '.join(map(str, set(pollutant_columns)))} ," f" FORMAT_DATETIME('%Y-%m-%d %H:%M:%S', {data_table}.timestamp) AS datetime " @@ -748,6 +763,8 @@ def get_downloadable_events( pm2_5="$pm2_5.value", pm10="$pm10.value", no2="$no2.value", + co="$co.value", + nh3="$nh3.value", frequency=1, site_id={"$toString": "$site_id"}, ) @@ -759,7 +776,7 @@ def get_downloadable_events( site_id={"$first": "$site_id"}, ) .project( - site_id={"$toObjectId": "$site_id"}, time=1, pm2_5=1, pm10=1, no2=1 + site_id={"$toObjectId": "$site_id"}, time=1, pm2_5=1, pm10=1, no2=1, co=1, nh3=1 ) .lookup("sites", local_field="site_id", foreign_field="_id", col_as="site") .unwind("site") @@ -802,7 +819,7 @@ def get_averages_by_pollutant(self, start_date, end_date, pollutant): @cache.memoize() def get_averages_by_pollutant_from_bigquery(self, start_date, end_date, pollutant): - if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]: + if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co",'nh3']: raise Exception("Invalid pollutant") query = f""" @@ -824,7 +841,7 @@ def get_averages_by_pollutant_from_bigquery(self, start_date, end_date, pollutan def get_device_averages_from_bigquery( self, start_date, end_date, pollutant, devices ): - if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]: + if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]: raise Exception("Invalid pollutant") query = f""" @@ -847,7 +864,7 @@ def get_device_averages_from_bigquery( def get_device_readings_from_bigquery( self, start_date, end_date, pollutant, devices ): - if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]: + if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]: raise Exception("Invalid pollutant") query = f""" @@ -996,7 +1013,7 @@ def get_d3_chart_events(self, sites, start_date, end_date, pollutant, frequency) def get_d3_chart_events_v2( self, sites, start_date, end_date, pollutant, frequency, tenant ): - if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]: + if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]: raise Exception("Invalid pollutant") columns = [ @@ -1078,6 +1095,8 @@ def get_events(self, sites, start_date, end_date, frequency): **{"pm2_5.value": 1}, **{"pm10.value": 1}, **{"no2.value": 1}, + **{"co.value": 1}, + **{"nh3.value": 1}, frequency=1, site_id={"$toString": "$site_id"}, sites={"name": 1, "description": 1, "generated_name": 1}, @@ -1089,6 +1108,8 @@ def get_events(self, sites, start_date, end_date, frequency): pm2_5={"$avg": "$pm2_5.value"}, pm10={"$avg": "$pm10.value"}, no2={"$avg": "$no2.value"}, + co={"$avg": "$co.value"}, + nh3={"$avg": "$nh3.value"}, sites={"$first": "$sites"}, ) .sort(**{"_id.time": 1}) diff --git a/src/analytics/api/utils/pollutants/pm_25.py b/src/analytics/api/utils/pollutants/pm_25.py index 9527caf00f..f447d253ce 100644 --- a/src/analytics/api/utils/pollutants/pm_25.py +++ b/src/analytics/api/utils/pollutants/pm_25.py @@ -14,6 +14,9 @@ "pm2_5": 88500, "pm10": 85101, "no2": 42602, + "co":42602, + "nh3": 42602, + } # TODO: Verify that these are the right units @@ -21,16 +24,24 @@ "pm2_5": "001", "pm10": "001", "no2": "008", + "co": "009", + "nh3": "009", } AQCSV_DATA_STATUS_MAPPER = { "pm2_5_calibrated_value": 1, "pm10_calibrated_value": 1, "no2_calibrated_value": 1, + "co_calibrated_value": 1, + "nh3_calibrated_value": 1, "pm2_5_raw_value": 0, "pm10_raw_value": 0, "no2_raw_value": 0, + "co_raw_value": 0, + "nh3_raw_value": 0, + # only used when frequency is raw + "no2": 0, "pm2_5": 0, "s1_pm2_5": 0, @@ -38,6 +49,8 @@ "s1_pm10": 0, "s2_pm2_5": 0, "s2_pm10": 0, + "co": 0, + "nh3": 0, } @@ -52,6 +65,8 @@ "pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"], "pm10": ["pm10_calibrated_value", "pm10_raw_value"], "no2": ["no2_calibrated_value", "no2_raw_value"], + "co": ["co_calibrated_value", "co_raw_value"], + "nh3": ["nh3_calibrated_value", "nh3_raw_value"], } BIGQUERY_FREQUENCY_MAPPER = { @@ -59,17 +74,22 @@ "pm2_5": ["pm2_5", "s1_pm2_5", "s2_pm2_5"], "pm10": ["pm10", "s1_pm10", "s2_pm10"], "no2": ["no2"], - "voc": ["voc"], + "co": ["co"], + "nh3": ["nh3"], }, "daily": { "pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"], "pm10": ["pm10_calibrated_value", "pm10_raw_value"], "no2": ["no2_calibrated_value", "no2_raw_value"], + "co": ["co_calibrated_value", "co_raw_value"], + "nh3": ["nh3_calibrated_value", "nh3_raw_value"], }, "hourly": { "pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"], "pm10": ["pm10_calibrated_value", "pm10_raw_value"], "no2": ["no2_calibrated_value", "no2_raw_value"], + "co": ["co_calibrated_value", "co_raw_value"], + "nh3": ["nh3_calibrated_value", "nh3_raw_value"], }, } @@ -113,7 +133,25 @@ "All": [0, 2049], } +CO_CATEGORY = { + "Good": [0, 4.4], + "Moderate": [4.5, 9.4], + "UHFSG": [9.5, 12.4], + "Unhealthy": [12.5, 15.4], + "VeryUnhealthy": [15.5,30.4], + "Hazardous": [30.5,50.4], + "All": [0, 50.4], +} +NH3_CATEGORY = { + "Good": [0, 200], + "Moderate": [201,400], + "UHFSG": [401, 800], + "Unhealthy": [801, 1200], + "VeryUnhealthy": [1201,1800], + "Hazardous": [1801,2500], + "All": [0, 2500], +} def set_pm25_category_background(pm25_value): keys = sorted(PM_25_COLOR_MAPPER.keys(), reverse=True) @@ -150,7 +188,7 @@ def get_pollutant_category(value, pollutant): Returns: a string representing the category og the value """ - mapper = {"pm2_5": PM_25_CATEGORY, "pm10": PM_10_CATEGORY, "no2": NO2_CATEGORY} + mapper = {"pm2_5": PM_25_CATEGORY, "pm10": PM_10_CATEGORY, "no2": NO2_CATEGORY, "co": CO_CATEGORY,"nh3": NH3_CATEGORY} try: category_mapper = dict(mapper[pollutant]) diff --git a/src/analytics/api/views/data.py b/src/analytics/api/views/data.py index fd667757c1..cd7825dc69 100644 --- a/src/analytics/api/views/data.py +++ b/src/analytics/api/views/data.py @@ -54,7 +54,7 @@ class DataExportResource(Resource): "airqlouds|optional:list", ) def post(self): - valid_pollutants = ["pm2_5", "pm10", "no2","voc"] + valid_pollutants = ["pm2_5", "pm10", "no2","voc","co","nh3"] valid_download_types = ["csv", "json"] valid_output_formats = ["airqo-standard", "aqcsv"] valid_frequencies = ["hourly", "daily", "raw"] @@ -199,7 +199,7 @@ class DataExportV2Resource(Resource): "meta_data|optional:dict", ) def post(self): - valid_pollutants = ["pm2_5", "pm10", "no2","voc"] + valid_pollutants = ["pm2_5", "pm10", "no2","voc","co","nh3"] valid_export_formats = ["csv", "json"] valid_output_formats = ["airqo-standard", "aqcsv"] valid_frequencies = ["hourly", "daily", "raw"] From 1b0d1f64f3be429fa0e91c4e44d2e86273ba4c9d Mon Sep 17 00:00:00 2001 From: wabinyai Date: Mon, 8 Jul 2024 14:23:09 +0300 Subject: [PATCH 3/3] gaseous --- src/analytics/api/utils/pollutants/pm_25.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/analytics/api/utils/pollutants/pm_25.py b/src/analytics/api/utils/pollutants/pm_25.py index f447d253ce..6b58cad33f 100644 --- a/src/analytics/api/utils/pollutants/pm_25.py +++ b/src/analytics/api/utils/pollutants/pm_25.py @@ -201,3 +201,4 @@ def get_pollutant_category(value, pollutant): return key return "Unknown" +