diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml index 98861f66e8..8d3b5292bc 100644 --- a/k8s/analytics/values-prod.yaml +++ b/k8s/analytics/values-prod.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job - tag: prod-3d3f6c14-1733315928 + tag: prod-3c9423e2-1733862695 api: name: airqo-analytics-api label: analytics-api diff --git a/k8s/analytics/values-stage.yaml b/k8s/analytics/values-stage.yaml index 6c2109404c..114d458f1a 100644 --- a/k8s/analytics/values-stage.yaml +++ b/k8s/analytics/values-stage.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-stage-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-devices-summary-job - tag: stage-014ccd0f-1733315863 + tag: stage-dd764c29-1733849460 api: name: airqo-stage-analytics-api label: sta-alytics-api diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml index b4c32c4d56..a359fb2e9b 100644 --- a/k8s/device-registry/values-prod.yaml +++ b/k8s/device-registry/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-device-registry-api - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/device-registry/values-stage.yaml b/k8s/device-registry/values-stage.yaml index c3d8dccf77..a92ee18671 100644 --- a/k8s/device-registry/values-stage.yaml +++ b/k8s/device-registry/values-stage.yaml @@ -6,7 +6,7 @@ app: replicaCount: 2 image: repository: eu.gcr.io/airqo-250220/airqo-stage-device-registry-api - tag: stage-dfe6eb16-1733832983 + tag: stage-9f584165-1733862604 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml index 8fd10c0c0c..d941e8d9e8 100644 --- a/k8s/exceedance/values-prod-airqo.yaml +++ b/k8s/exceedance/values-prod-airqo.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/airqo-exceedance-job - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml index 72030a5345..dfc47322a5 100644 --- a/k8s/exceedance/values-prod-kcca.yaml +++ b/k8s/exceedance/values-prod-kcca.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/kcca-exceedance-job - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml index 70cf680436..ef7a86c858 100644 --- a/k8s/predict/values-prod.yaml +++ b/k8s/predict/values-prod.yaml @@ -7,7 +7,7 @@ images: predictJob: eu.gcr.io/airqo-250220/airqo-predict-job trainJob: eu.gcr.io/airqo-250220/airqo-train-job predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 api: name: airqo-prediction-api label: prediction-api diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml index 3e448286e8..149d92b402 100644 --- a/k8s/spatial/values-prod.yaml +++ b/k8s/spatial/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-spatial-api - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml index 6460bba388..759c1c62f3 100644 --- a/k8s/website/values-prod.yaml +++ b/k8s/website/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-website-api - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml index 723152603a..c736740f1d 100644 --- a/k8s/workflows/values-prod.yaml +++ b/k8s/workflows/values-prod.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-redis containers: eu.gcr.io/airqo-250220/airqo-workflows - tag: prod-ee15b958-1733833086 + tag: prod-3c9423e2-1733862695 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/src/analytics/api/models/events.py b/src/analytics/api/models/events.py index 4ac9e2febb..a3883752f2 100644 --- a/src/analytics/api/models/events.py +++ b/src/analytics/api/models/events.py @@ -413,6 +413,7 @@ def download_from_bigquery( frequency (str): Data frequency (e.g., 'raw', 'daily', 'hourly'). pollutants (list): List of pollutants to include in the data. data_type (str): Type of data ('raw' or 'aggregated'). + filter_columns(list) weather_fields (list): List of weather fields to retrieve. Returns: @@ -438,14 +439,11 @@ def download_from_bigquery( weather_columns = [] for pollutant in pollutants: - if pollutant == "raw": - key = pollutant - else: - key = f"{pollutant}_{data_type}" - + key = f"{pollutant}_{data_type}" pollutant_mapping = BIGQUERY_FREQUENCY_MAPPER.get(frequency, {}).get( key, [] ) + pollutant_columns.extend( cls.get_columns( cls, @@ -459,6 +457,10 @@ def download_from_bigquery( # TODO Clean up by use using `get_columns` helper method if pollutant in {"pm2_5", "pm10", "no2"}: + if data_type == "raw": + # Add dummy column to fix union column number missmatch. + bam_pollutant_columns.append("-1 as pm2_5") + if frequency in ["weekly", "monthly", "yearly"]: bam_pollutant_columns.extend( [f"ROUND(AVG({pollutant}), {decimal_places}) AS {key}_value"] @@ -467,6 +469,7 @@ def download_from_bigquery( bam_pollutant_columns.extend( [f"ROUND({pollutant}, {decimal_places}) AS {key}_value"] ) + # TODO Fix query when weather data is included. Currently failing if weather_fields: for field in weather_fields: @@ -536,12 +539,55 @@ def download_from_bigquery( drop_columns.append("datetime") sorting_cols.append("datetime") + if data_type == "raw": + cls.simple_data_cleaning(dataframe) + dataframe.drop_duplicates(subset=drop_columns, inplace=True, keep="first") dataframe.sort_values(sorting_cols, ascending=True, inplace=True) dataframe["frequency"] = frequency dataframe = dataframe.replace(np.nan, None) return dataframe + @classmethod + def simple_data_cleaning(cls, data: pd.DataFrame) -> pd.DataFrame: + """ + Perform data cleaning on a pandas DataFrame to handle specific conditions + related to "pm2_5" and "pm2_5_raw_value" columns. + + The cleaning process includes: + 1. Ensuring correct numeric data types for "pm2_5" and "pm2_5_raw_value". + 2. Removing "pm2_5" values where "pm2_5_raw_value" has data. + 3. Dropping the "pm2_5_raw_value" column if it has no data at all. + 4. Retaining "pm2_5" values where "pm2_5_raw_value" has no data, and removing + "pm2_5" values where "pm2_5_raw_value" has data. + 5. Dropping any column (including "pm2_5" and "pm2_5_raw_value") if it is + entirely empty. + + Args: + cls: Class reference (used in classmethods). + data (pd.DataFrame): Input pandas DataFrame with "pm2_5" and + "pm2_5_raw_value" columns. + + Returns: + pd.DataFrame: Cleaned DataFrame with updates applied in place. + + """ + data["pm2_5_raw_value"] = pd.to_numeric( + data["pm2_5_raw_value"], errors="coerce" + ) + data["pm2_5"] = pd.to_numeric(data["pm2_5"], errors="coerce") + + data.loc[~data["pm2_5_raw_value"].isna(), "pm2_5"] = np.nan + + if data["pm2_5_raw_value"].isna().all(): + data.drop(columns=["pm2_5_raw_value"], inplace=True) + + data["pm2_5"] = data["pm2_5"].where(data["pm2_5_raw_value"].isna(), np.nan) + + data.dropna(how="all", axis=1, inplace=True) + + return data + @classmethod def data_export_query( cls, @@ -1273,9 +1319,7 @@ def get_d3_chart_events(self, sites, start_date, end_date, pollutant, frequency) ) @cache.memoize() - def get_d3_chart_events_v2( - self, sites, start_date, end_date, pollutant, frequency, tenant - ): + def get_d3_chart_events_v2(self, sites, start_date, end_date, pollutant, frequency): if pollutant not in ["pm2_5", "pm10", "no2", "pm1"]: raise Exception("Invalid pollutant") @@ -1293,7 +1337,6 @@ def get_d3_chart_events_v2( JOIN {self.BIGQUERY_SITES} ON {self.BIGQUERY_SITES}.id = {self.BIGQUERY_EVENTS}.site_id WHERE {self.BIGQUERY_EVENTS}.timestamp >= '{start_date}' AND {self.BIGQUERY_EVENTS}.timestamp <= '{end_date}' - AND {self.BIGQUERY_EVENTS}.tenant = '{tenant}' AND `airqo-250220.metadata.sites`.id in UNNEST({sites}) """ diff --git a/src/analytics/api/utils/data_formatters.py b/src/analytics/api/utils/data_formatters.py index bace9559de..d1f5eda567 100644 --- a/src/analytics/api/utils/data_formatters.py +++ b/src/analytics/api/utils/data_formatters.py @@ -328,6 +328,31 @@ def filter_non_private_sites(filter_type: str, sites: List[str]) -> Dict[str, An logger.exception(f"Error while filtering non private devices {rex}") +def validate_network(network_name: str) -> bool: + """ + Validate if a given network name exists in the list of networks. + + Args: + network_name (str): The name of the network to validate. + + Returns: + bool: True if the network name exists, False otherwise. + """ + if not network_name: + return False + + endpoint: str = "/users/networks" + airqo_requests = AirQoRequests() + response = airqo_requests.request(endpoint=endpoint, method="get") + + if response and "networks" in response: + networks = response["networks"] + # TODO Could add an active network filter + return any(network.get("net_name") == network_name for network in networks) + + return False + + def filter_non_private_devices(filter_type: str, devices: List[str]) -> Dict[str, Any]: """ FilterS out private device IDs from a provided array of device IDs. diff --git a/src/analytics/api/utils/pollutants/pm_25.py b/src/analytics/api/utils/pollutants/pm_25.py index 6e3f2a986b..9c16abf988 100644 --- a/src/analytics/api/utils/pollutants/pm_25.py +++ b/src/analytics/api/utils/pollutants/pm_25.py @@ -56,19 +56,14 @@ COMMON_POLLUTANT_MAPPING = { "pm2_5_calibrated": ["pm2_5_calibrated_value"], - "pm2_5_raw": ["pm2_5_raw_value"], + "pm2_5_raw": ["pm2_5_raw_value", "pm2_5"], "pm10_calibrated": ["pm10_calibrated_value"], - "pm10_raw": ["pm10_raw_value"], + "pm10_raw": ["pm10_raw_value", "pm10"], "no2_calibrated": ["no2_calibrated_value"], "no2_raw": ["no2_raw_value"], } BIGQUERY_FREQUENCY_MAPPER = { - "raw": { - "pm2_5": ["pm2_5", "s1_pm2_5", "s2_pm2_5"], - "pm10": ["pm10", "s1_pm10", "s2_pm10"], - "no2": ["no2"], - }, "daily": COMMON_POLLUTANT_MAPPING, "hourly": COMMON_POLLUTANT_MAPPING, "weekly": COMMON_POLLUTANT_MAPPING, diff --git a/src/analytics/api/views/dashboard.py b/src/analytics/api/views/dashboard.py index 62d31bc01e..3e4429495e 100644 --- a/src/analytics/api/views/dashboard.py +++ b/src/analytics/api/views/dashboard.py @@ -206,7 +206,6 @@ def _get_validated_filter(self, json_data): return filter_type, validated_data, error_message def post(self): - tenant = request.args.get("tenant", "airqo") json_data = request.get_json() @@ -219,20 +218,16 @@ def post(self): except Exception as e: logger.exception(f"An error has occured; {e}") - sites = filter_non_private_sites("sites", json_data.get("sites", {})).get( - "sites", [] - ) - start_date = json_data["startDate"] end_date = json_data["endDate"] frequency = json_data["frequency"] pollutant = json_data["pollutant"] chart_type = json_data["chartType"] - events_model = EventsModel(tenant) - # data = events_model.get_d3_chart_events(sites, start_date, end_date, pollutant, frequency) + events_model = EventsModel("airqo") + data = events_model.get_d3_chart_events_v2( - filter_value, start_date, end_date, pollutant, frequency, tenant + filter_value, start_date, end_date, pollutant, frequency ) if chart_type.lower() == "pie": diff --git a/src/analytics/api/views/data.py b/src/analytics/api/views/data.py index 6038e80497..07ef72c0a2 100644 --- a/src/analytics/api/views/data.py +++ b/src/analytics/api/views/data.py @@ -166,6 +166,7 @@ def post(self): data_frame.drop( columns=[ "site_id", + "timestamp", ], inplace=True, ) diff --git a/src/device-registry/config/global/db-projections.js b/src/device-registry/config/global/db-projections.js index 8282998021..37f14abffc 100644 --- a/src/device-registry/config/global/db-projections.js +++ b/src/device-registry/config/global/db-projections.js @@ -135,7 +135,7 @@ const dbProjections = { lat_long: 1, country: 1, network: 1, - group: 1, + groups: 1, data_provider: 1, district: 1, sub_county: 1, @@ -379,7 +379,7 @@ const dbProjections = { mobility: 1, status: 1, network: 1, - group: 1, + groups: 1, api_code: 1, serial_number: 1, authRequired: 1, @@ -591,7 +591,7 @@ const dbProjections = { shape: 1, createdAt: 1, network: 1, - group: 1, + groups: 1, sites: "$sites", numberOfSites: { $cond: { @@ -675,7 +675,7 @@ const dbProjections = { name: 1, description: 1, cohort_tags: 1, - group: 1, + groups: 1, createdAt: 1, visibility: 1, cohort_codes: 1, @@ -817,7 +817,7 @@ const dbProjections = { airqloud_tags: 1, isCustom: 1, network: 1, - group: 1, + groups: 1, metadata: 1, center_point: 1, sites: "$sites", @@ -1065,7 +1065,7 @@ const dbProjections = { date: 1, description: 1, network: 1, - group: 1, + groups: 1, activityType: 1, maintenanceType: 1, recallType: 1, diff --git a/src/device-registry/models/Activity.js b/src/device-registry/models/Activity.js index 0e5863949e..9cd5ada002 100644 --- a/src/device-registry/models/Activity.js +++ b/src/device-registry/models/Activity.js @@ -40,8 +40,8 @@ const activitySchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, activityType: { type: String, trim: true }, @@ -70,7 +70,7 @@ activitySchema.methods = { _id: this._id, device: this.device, network: this.network, - group: this.group, + groups: this.groups, date: this.date, description: this.description, activityType: this.activityType, diff --git a/src/device-registry/models/Airqloud.js b/src/device-registry/models/Airqloud.js index beacd653db..1be15e8683 100644 --- a/src/device-registry/models/Airqloud.js +++ b/src/device-registry/models/Airqloud.js @@ -113,8 +113,8 @@ const airqloudSchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, airqloud_tags: { @@ -152,7 +152,7 @@ airqloudSchema.methods.toJSON = function() { name: this.name, long_name: this.long_name, network: this.network, - group: this.group, + groups: this.groups, description: this.description, airqloud_tags: this.airqloud_tags, admin_level: this.admin_level, diff --git a/src/device-registry/models/Cohort.js b/src/device-registry/models/Cohort.js index 788811e60a..a00884dacd 100644 --- a/src/device-registry/models/Cohort.js +++ b/src/device-registry/models/Cohort.js @@ -17,8 +17,8 @@ const cohortSchema = new Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, name: { @@ -81,7 +81,7 @@ cohortSchema.methods.toJSON = function() { cohort_tags, cohort_codes, network, - group, + groups, visibility, } = this; return { @@ -92,7 +92,7 @@ cohortSchema.methods.toJSON = function() { cohort_tags, cohort_codes, network, - group, + groups, }; }; @@ -202,7 +202,7 @@ cohortSchema.statics.list = async function( name: 1, createdAt: 1, network: 1, - group: 1, + groups: 1, devices: { $cond: { if: { $eq: [{ $size: "$devices" }, 0] }, @@ -222,7 +222,7 @@ cohortSchema.statics.list = async function( name: { $first: "$name" }, createdAt: { $first: "$createdAt" }, network: { $first: "$network" }, - group: { $first: "$group" }, + groups: { $first: "$groups" }, devices: { $first: "$devices" }, }) .skip(skip ? parseInt(skip) : 0) @@ -240,7 +240,7 @@ cohortSchema.statics.list = async function( name: cohort.name, network: cohort.network, createdAt: cohort.createdAt, - group: cohort.group, + groups: cohort.groups, numberOfDevices: cohort.devices ? cohort.devices.length : 0, devices: cohort.devices ? cohort.devices @@ -250,7 +250,7 @@ cohortSchema.statics.list = async function( status: device.status, name: device.name, network: device.network, - group: device.group, + groups: device.groups, device_number: device.device_number, description: device.description, long_name: device.long_name, diff --git a/src/device-registry/models/Device.js b/src/device-registry/models/Device.js index a5b10c7500..0f5b08749d 100644 --- a/src/device-registry/models/Device.js +++ b/src/device-registry/models/Device.js @@ -74,8 +74,8 @@ const deviceSchema = new mongoose.Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, serial_number: { @@ -237,6 +237,20 @@ deviceSchema.plugin(uniqueValidator, { message: `{VALUE} must be unique!`, }); +const checkDuplicates = (arr, fieldName) => { + const duplicateValues = arr.filter( + (value, index, self) => self.indexOf(value) !== index + ); + + if (duplicateValues.length > 0) { + return new HttpError( + `Duplicate values found in ${fieldName} array.`, + httpStatus.BAD_REQUEST + ); + } + return null; +}; + deviceSchema.pre( [ "update", @@ -326,18 +340,16 @@ deviceSchema.pre( this.device_codes.push(this.serial_number); } - // Check for duplicate values in cohorts array - const duplicateValues = this.cohorts.filter( - (value, index, self) => self.indexOf(value) !== index - ); + // Check for duplicates in cohorts + const cohortsDuplicateError = checkDuplicates(this.cohorts, "cohorts"); + if (cohortsDuplicateError) { + return next(cohortsDuplicateError); + } - if (duplicateValues.length > 0) { - return next( - new HttpError( - "Duplicate values found in cohorts array.", - httpStatus.BAD_REQUEST - ) - ); + // Check for duplicates in groups + const groupsDuplicateError = checkDuplicates(this.groups, "groups"); + if (groupsDuplicateError) { + return next(groupsDuplicateError); } } @@ -371,28 +383,20 @@ deviceSchema.pre( updateData.access_code = access_code.toUpperCase(); } - // Handle $addToSet for device_codes, previous_sites, and pictures - const addToSetUpdates = {}; - - if (updateData.device_codes) { - addToSetUpdates.device_codes = { $each: updateData.device_codes }; - delete updateData.device_codes; // Remove from main update object - } - - if (updateData.previous_sites) { - addToSetUpdates.previous_sites = { $each: updateData.previous_sites }; - delete updateData.previous_sites; // Remove from main update object - } - - if (updateData.pictures) { - addToSetUpdates.pictures = { $each: updateData.pictures }; - delete updateData.pictures; // Remove from main update object - } - - // If there are any $addToSet updates, merge them into the main update object - if (Object.keys(addToSetUpdates).length > 0) { - updateData.$addToSet = addToSetUpdates; - } + // Handle array fields using $addToSet + const arrayFieldsToAddToSet = [ + "device_codes", + "previous_sites", + "groups", + "pictures", + ]; + arrayFieldsToAddToSet.forEach((field) => { + if (updateData[field]) { + updateData.$addToSet = updateData.$addToSet || {}; + updateData.$addToSet[field] = { $each: updateData[field] }; + delete updateData[field]; + } + }); next(); } catch (error) { @@ -415,7 +419,7 @@ deviceSchema.methods = { alias: this.alias, mobility: this.mobility, network: this.network, - group: this.group, + groups: this.groups, api_code: this.api_code, serial_number: this.serial_number, authRequired: this.authRequired, diff --git a/src/device-registry/models/Grid.js b/src/device-registry/models/Grid.js index 53e8e5c027..231704c5b2 100644 --- a/src/device-registry/models/Grid.js +++ b/src/device-registry/models/Grid.js @@ -43,8 +43,8 @@ const gridSchema = new Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, geoHash: { @@ -121,7 +121,7 @@ gridSchema.methods.toJSON = function() { name, long_name, network, - group, + groups, visibility, description, grid_tags, @@ -139,7 +139,7 @@ gridSchema.methods.toJSON = function() { description, grid_tags, network, - group, + groups, admin_level, grid_codes, centers, diff --git a/src/device-registry/models/Location.js b/src/device-registry/models/Location.js index b37bd7c349..b5867155eb 100644 --- a/src/device-registry/models/Location.js +++ b/src/device-registry/models/Location.js @@ -84,6 +84,10 @@ const locationSchema = new Schema( type: String, trim: true, }, + groups: { + type: [String], + trim: true, + }, location_tags: { type: Array, default: [], @@ -124,6 +128,7 @@ locationSchema.methods = { isCustom: this.isCustom, location: this.location, network: this.network, + groups: this.groups, metadata: this.metadata, }; }, @@ -190,6 +195,7 @@ locationSchema.statics = { isCustom: 1, metadata: 1, network: 1, + groups: 1, sites: "$sites", }; @@ -200,6 +206,7 @@ locationSchema.statics = { admin_level: 1, description: 1, network: 1, + groups: 1, metadata: 1, }; @@ -306,6 +313,7 @@ locationSchema.statics = { description: 1, admin_level: 1, network: 1, + groups: 1, isCustom: 1, metadata: 1, }, diff --git a/src/device-registry/models/Photo.js b/src/device-registry/models/Photo.js index 78fb7c6679..385be95009 100644 --- a/src/device-registry/models/Photo.js +++ b/src/device-registry/models/Photo.js @@ -18,8 +18,8 @@ const photoSchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, device_id: { @@ -83,7 +83,7 @@ photoSchema.methods = { tags: this.tags, name: this.name, network: this.network, - group: this.group, + groups: this.groups, image_url: this.image_url, device_id: this.device_id, site_id: this.site_id, @@ -158,7 +158,7 @@ photoSchema.statics = { description: 1, metadata: 1, network: 1, - group: 1, + groups: 1, }) .skip(skip ? skip : 0) .limit(limit ? limit : 1000) diff --git a/src/device-registry/models/Site.js b/src/device-registry/models/Site.js index 101b5fa1a0..3e0946db40 100644 --- a/src/device-registry/models/Site.js +++ b/src/device-registry/models/Site.js @@ -75,8 +75,8 @@ const siteSchema = new Schema( trim: true, required: [true, "network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, data_provider: { @@ -364,6 +364,20 @@ const siteSchema = new Schema( } ); +const checkDuplicates = (arr, fieldName) => { + const duplicateValues = arr.filter( + (value, index, self) => self.indexOf(value) !== index + ); + + if (duplicateValues.length > 0) { + return new HttpError( + `Duplicate values found in ${fieldName} array.`, + httpStatus.BAD_REQUEST + ); + } + return null; +}; + siteSchema.pre( ["updateOne", "findOneAndUpdate", "updateMany", "update", "save"], function(next) { @@ -411,6 +425,7 @@ siteSchema.pre( "land_use", "site_codes", "airqlouds", + "groups", "grids", ]; arrayFieldsToAddToSet.forEach((field) => { @@ -442,12 +457,16 @@ siteSchema.pre( if (this[field]) this.site_codes.push(this[field]); }); - // Check for duplicate grid values - const duplicateValues = this.grids.filter( - (value, index, self) => self.indexOf(value) !== index - ); - if (duplicateValues.length > 0) { - return next(new Error("Duplicate values found in grids array.")); + // Check for duplicates in grids + const gridsDuplicateError = checkDuplicates(this.grids, "grids"); + if (gridsDuplicateError) { + return next(gridsDuplicateError); + } + + // Check for duplicates in groups + const groupsDuplicateError = checkDuplicates(this.groups, "groups"); + if (groupsDuplicateError) { + return next(groupsDuplicateError); } } @@ -473,7 +492,7 @@ siteSchema.methods = { generated_name: this.generated_name, search_name: this.search_name, network: this.network, - group: this.group, + groups: this.groups, data_provider: this.data_provider, location_name: this.location_name, formatted_name: this.formatted_name, diff --git a/src/device-registry/routes/v2/cohorts.js b/src/device-registry/routes/v2/cohorts.js index e2d5a07a2b..8f047c0054 100644 --- a/src/device-registry/routes/v2/cohorts.js +++ b/src/device-registry/routes/v2/cohorts.js @@ -128,6 +128,15 @@ router.put( .trim() .isBoolean() .withMessage("visibility must be Boolean"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .optional() .notEmpty() @@ -169,6 +178,15 @@ router.post( .trim() .optional() .notEmpty(), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .trim() .exists() diff --git a/src/device-registry/routes/v2/grids.js b/src/device-registry/routes/v2/grids.js index f70426a4b6..a8678a7a8e 100644 --- a/src/device-registry/routes/v2/grids.js +++ b/src/device-registry/routes/v2/grids.js @@ -195,6 +195,15 @@ router.post( .withMessage( "admin_level values include but not limited to: province, state, village, county, etc. Update your GLOBAL configs" ), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .trim() .optional() @@ -371,6 +380,15 @@ router.put( .optional() .notEmpty() .withMessage("the description should not be empty if provided"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .optional() .notEmpty() diff --git a/src/device-registry/routes/v2/sites.js b/src/device-registry/routes/v2/sites.js index 8d3198e89e..8117e500d0 100644 --- a/src/device-registry/routes/v2/sites.js +++ b/src/device-registry/routes/v2/sites.js @@ -486,6 +486,15 @@ router.post( .bail() .notEmpty() .withMessage("the site_tags should not be empty"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("airqlouds") .optional() .custom((value) => { @@ -860,6 +869,15 @@ router.put( .withMessage( "Invalid site_category format, crosscheck the types or content of all the provided nested fields. latitude, longitude & search_radius should be numbers. tags should be an array of strings. category, search_tags & search_radius are required fields" ), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), ], ]), siteController.update diff --git a/src/device-registry/utils/generate-filter.js b/src/device-registry/utils/generate-filter.js index e6320d57b3..e8366e409f 100644 --- a/src/device-registry/utils/generate-filter.js +++ b/src/device-registry/utils/generate-filter.js @@ -1116,7 +1116,7 @@ const generateFilter = { // } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1238,7 +1238,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1370,7 +1370,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1437,7 +1437,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1500,7 +1500,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1597,7 +1597,7 @@ const generateFilter = { } }, locations: (req, next) => { - let { id, name, admin_level, summary, network } = { + let { id, name, admin_level, summary, network, group } = { ...req.query, ...req.params, }; @@ -1623,6 +1623,14 @@ const generateFilter = { ); } + if (group) { + filter.groups = handlePredefinedValueMatch( + group, + constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, + { matchCombinations: true } + ); + } + if (admin_level) { filter["admin_level"] = admin_level; } @@ -1671,7 +1679,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1759,7 +1767,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } diff --git a/src/device-registry/utils/scripts/new-bulk-script.js b/src/device-registry/utils/scripts/new-bulk-script.js new file mode 100644 index 0000000000..37b7f4c23b --- /dev/null +++ b/src/device-registry/utils/scripts/new-bulk-script.js @@ -0,0 +1,71 @@ +const axios = require("axios"); +const isEmpty = require("is-empty"); + +const url = "http://localhost:3000/api/v2/devices/sites"; +const config = { + headers: { + Authorization: "", + }, +}; + +const NETWORK_MAPPINGS = { + // iqair: "permian-health", + // usembassy: "us-embassy", + // urbanbetter: "urban-better", + // kcca: "kcca", + // airqo: "airqo", + // Add more mappings as needed +}; + +const DEFAULT_GROUP = "unknown"; + +axios + .get(url, config) + .then(async (response) => { + const groups = response.data.sites + .map((site) => { + // Look up the group based on network, with a fallback to a default + const group = NETWORK_MAPPINGS[site.network] || DEFAULT_GROUP; + + // Optionally log devices with unknown networks + if (group === DEFAULT_GROUP) { + console.log( + `Unrecognized network for device ${site.name}: ${site.network}` + ); + } + + return group; + }) + // Remove any 'unknown' groups if you want only mapped networks + .filter((group) => group !== DEFAULT_GROUP); + + console.log("the data:"); + console.dir({ groups }); + + // Process devices in batches + for (let i = 0; i < response.data.sites.length; i += 10) { + const batch = response.data.sites.slice(i, i + 10); + + for (const site of batch) { + const group = NETWORK_MAPPINGS[site.network] || DEFAULT_GROUP; + + if (group !== DEFAULT_GROUP) { + const url = `http://localhost:3000/api/v2/devices/sites?id=${site._id}`; + const data = { groups: [group] }; + // console.log("the data:"); + // console.dir(data); + + try { + // Uncomment if you want to make the PUT request + const putResponse = await axios.put(url, data, config); + console.log("PUT response:", putResponse.data); + } catch (error) { + console.error("PUT error:", error.message); + } + } + } + } + }) + .catch((error) => { + console.error("GET error:", error); + }); diff --git a/src/device-registry/validators/device.validators.js b/src/device-registry/validators/device.validators.js index e0b4916f55..b08baea97d 100644 --- a/src/device-registry/validators/device.validators.js +++ b/src/device-registry/validators/device.validators.js @@ -126,6 +126,15 @@ const validateCreateDevice = [ .isInt() .withMessage("the generation should be an integer") .toInt(), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("mountType") .optional() .notEmpty() @@ -384,6 +393,15 @@ const validateUpdateDevice = [ .trim() .isBoolean() .withMessage("isActive must be Boolean"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("isRetired") .optional() .notEmpty()