Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gaseous Pollutants #3146

Closed
wants to merge 3 commits into from
Closed

Gaseous Pollutants #3146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0ec5b6a
Gaseous Pollutants
wabinyai Apr 15, 2024
fc4be80
database
wabinyai Apr 16, 2024
1b0d1f6
gaseous
wabinyai Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 28 additions & 7 deletions src/analytics/api/models/events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class EventsModel(BasePyMongoModel):
DEVICES_SUMMARY_TABLE = CONFIGURATIONS.DEVICES_SUMMARY_TABLE

def __init__(self, tenant):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @wabinyai , thanks for this. Do these gases also follow the AQI?

self.limit_mapper = {"pm2_5": 500.5, "pm10": 604.5, "no2": 2049}
self.limit_mapper = {"pm2_5": 500.5, "pm10": 604.5, "no2": 2049, "co": 50.5 ,"nh3":2500}
super().__init__(tenant, collection_name="events")

@classmethod
Expand Down Expand Up @@ -95,6 +95,14 @@ def download_from_bigquery(
bam_pollutant_columns.extend(
["no2 as no2_raw_value", "no2 as no2_calibrated_value"]
)
elif pollutant == "co":
bam_pollutant_columns.extend(
["co as co_raw_value", "co as co_calibrated_value"]
)
elif pollutant == "nh3":
bam_pollutant_columns.extend(
["nh3 as co_raw_value", "nh3 as co_calibrated_value"]
)

pollutants_query = (
f" SELECT {', '.join(map(str, set(pollutant_columns)))} ,"
Expand Down Expand Up @@ -315,7 +323,14 @@ def data_export_query(
bam_pollutant_columns.extend(
["no2 as no2_raw_value", "no2 as no2_calibrated_value"]
)

elif pollutant == "co":
bam_pollutant_columns.extend(
["co as co_raw_value", "co as co_calibrated_value"]
)
elif pollutant == "nh3":
bam_pollutant_columns.extend(
["nh3 as nh3_raw_value", "nh3 as nh3_calibrated_value"]
)
pollutants_query = (
f" SELECT {', '.join(map(str, set(pollutant_columns)))} ,"
f" FORMAT_DATETIME('%Y-%m-%d %H:%M:%S', {data_table}.timestamp) AS datetime "
Expand Down Expand Up @@ -748,6 +763,8 @@ def get_downloadable_events(
pm2_5="$pm2_5.value",
pm10="$pm10.value",
no2="$no2.value",
co="$co.value",
nh3="$nh3.value",
frequency=1,
site_id={"$toString": "$site_id"},
)
Expand All @@ -759,7 +776,7 @@ def get_downloadable_events(
site_id={"$first": "$site_id"},
)
.project(
site_id={"$toObjectId": "$site_id"}, time=1, pm2_5=1, pm10=1, no2=1
site_id={"$toObjectId": "$site_id"}, time=1, pm2_5=1, pm10=1, no2=1, co=1, nh3=1
)
.lookup("sites", local_field="site_id", foreign_field="_id", col_as="site")
.unwind("site")
Expand Down Expand Up @@ -802,7 +819,7 @@ def get_averages_by_pollutant(self, start_date, end_date, pollutant):

@cache.memoize()
def get_averages_by_pollutant_from_bigquery(self, start_date, end_date, pollutant):
if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]:
if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co",'nh3']:
raise Exception("Invalid pollutant")

query = f"""
Expand All @@ -824,7 +841,7 @@ def get_averages_by_pollutant_from_bigquery(self, start_date, end_date, pollutan
def get_device_averages_from_bigquery(
self, start_date, end_date, pollutant, devices
):
if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]:
if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]:
raise Exception("Invalid pollutant")

query = f"""
Expand All @@ -847,7 +864,7 @@ def get_device_averages_from_bigquery(
def get_device_readings_from_bigquery(
self, start_date, end_date, pollutant, devices
):
if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]:
if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]:
raise Exception("Invalid pollutant")

query = f"""
Expand Down Expand Up @@ -996,7 +1013,7 @@ def get_d3_chart_events(self, sites, start_date, end_date, pollutant, frequency)
def get_d3_chart_events_v2(
self, sites, start_date, end_date, pollutant, frequency, tenant
):
if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]:
if pollutant not in ["pm2_5", "pm10", "no2", "pm1","co","nh3"]:
raise Exception("Invalid pollutant")

columns = [
Expand Down Expand Up @@ -1078,6 +1095,8 @@ def get_events(self, sites, start_date, end_date, frequency):
**{"pm2_5.value": 1},
**{"pm10.value": 1},
**{"no2.value": 1},
**{"co.value": 1},
**{"nh3.value": 1},
frequency=1,
site_id={"$toString": "$site_id"},
sites={"name": 1, "description": 1, "generated_name": 1},
Expand All @@ -1089,6 +1108,8 @@ def get_events(self, sites, start_date, end_date, frequency):
pm2_5={"$avg": "$pm2_5.value"},
pm10={"$avg": "$pm10.value"},
no2={"$avg": "$no2.value"},
co={"$avg": "$co.value"},
nh3={"$avg": "$nh3.value"},
sites={"$first": "$sites"},
)
.sort(**{"_id.time": 1})
Expand Down
42 changes: 41 additions & 1 deletion src/analytics/api/utils/pollutants/pm_25.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,30 +14,43 @@
"pm2_5": 88500,
"pm10": 85101,
"no2": 42602,
"co":42602,
"nh3": 42602,

}

# TODO: Verify that these are the right units
AQCSV_UNIT_MAPPER = {
"pm2_5": "001",
"pm10": "001",
"no2": "008",
"co": "009",
"nh3": "009",
}

AQCSV_DATA_STATUS_MAPPER = {
"pm2_5_calibrated_value": 1,
"pm10_calibrated_value": 1,
"no2_calibrated_value": 1,
"co_calibrated_value": 1,
"nh3_calibrated_value": 1,
"pm2_5_raw_value": 0,
"pm10_raw_value": 0,
"no2_raw_value": 0,
"co_raw_value": 0,
"nh3_raw_value": 0,

# only used when frequency is raw

"no2": 0,
"pm2_5": 0,
"s1_pm2_5": 0,
"pm10": 0,
"s1_pm10": 0,
"s2_pm2_5": 0,
"s2_pm10": 0,
"co": 0,
"nh3": 0,
}


Expand All @@ -52,23 +65,31 @@
"pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"],
"pm10": ["pm10_calibrated_value", "pm10_raw_value"],
"no2": ["no2_calibrated_value", "no2_raw_value"],
"co": ["co_calibrated_value", "co_raw_value"],
"nh3": ["nh3_calibrated_value", "nh3_raw_value"],
}

BIGQUERY_FREQUENCY_MAPPER = {
"raw": {
"pm2_5": ["pm2_5", "s1_pm2_5", "s2_pm2_5"],
"pm10": ["pm10", "s1_pm10", "s2_pm10"],
"no2": ["no2"],
"co": ["co"],
"nh3": ["nh3"],
},
"daily": {
"pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"],
"pm10": ["pm10_calibrated_value", "pm10_raw_value"],
"no2": ["no2_calibrated_value", "no2_raw_value"],
"co": ["co_calibrated_value", "co_raw_value"],
"nh3": ["nh3_calibrated_value", "nh3_raw_value"],
},
"hourly": {
"pm2_5": ["pm2_5_calibrated_value", "pm2_5_raw_value"],
"pm10": ["pm10_calibrated_value", "pm10_raw_value"],
"no2": ["no2_calibrated_value", "no2_raw_value"],
"co": ["co_calibrated_value", "co_raw_value"],
"nh3": ["nh3_calibrated_value", "nh3_raw_value"],
},
}

Expand Down Expand Up @@ -112,7 +133,25 @@
"All": [0, 2049],
}

CO_CATEGORY = {
"Good": [0, 4.4],
"Moderate": [4.5, 9.4],
"UHFSG": [9.5, 12.4],
"Unhealthy": [12.5, 15.4],
"VeryUnhealthy": [15.5,30.4],
"Hazardous": [30.5,50.4],
"All": [0, 50.4],
}

NH3_CATEGORY = {
"Good": [0, 200],
"Moderate": [201,400],
"UHFSG": [401, 800],
"Unhealthy": [801, 1200],
"VeryUnhealthy": [1201,1800],
"Hazardous": [1801,2500],
"All": [0, 2500],
}
def set_pm25_category_background(pm25_value):
keys = sorted(PM_25_COLOR_MAPPER.keys(), reverse=True)

Expand Down Expand Up @@ -149,7 +188,7 @@ def get_pollutant_category(value, pollutant):
Returns: a string representing the category og the value
"""

mapper = {"pm2_5": PM_25_CATEGORY, "pm10": PM_10_CATEGORY, "no2": NO2_CATEGORY}
mapper = {"pm2_5": PM_25_CATEGORY, "pm10": PM_10_CATEGORY, "no2": NO2_CATEGORY, "co": CO_CATEGORY,"nh3": NH3_CATEGORY}

try:
category_mapper = dict(mapper[pollutant])
Expand All @@ -162,3 +201,4 @@ def get_pollutant_category(value, pollutant):
return key

return "Unknown"

4 changes: 2 additions & 2 deletions src/analytics/api/views/data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class DataExportResource(Resource):
"airqlouds|optional:list",
)
def post(self):
valid_pollutants = ["pm2_5", "pm10", "no2"]
valid_pollutants = ["pm2_5", "pm10", "no2","voc","co","nh3"]
valid_download_types = ["csv", "json"]
valid_output_formats = ["airqo-standard", "aqcsv"]
valid_frequencies = ["hourly", "daily", "raw"]
Expand Down Expand Up @@ -199,7 +199,7 @@ class DataExportV2Resource(Resource):
"meta_data|optional:dict",
)
def post(self):
valid_pollutants = ["pm2_5", "pm10", "no2"]
valid_pollutants = ["pm2_5", "pm10", "no2","voc","co","nh3"]
valid_export_formats = ["csv", "json"]
valid_output_formats = ["airqo-standard", "aqcsv"]
valid_frequencies = ["hourly", "daily", "raw"]
Expand Down
Loading