From 99b38f93f39383d15406714825f6014474953c1d Mon Sep 17 00:00:00 2001 From: baalmart Date: Tue, 10 Dec 2024 18:05:45 +0300 Subject: [PATCH 1/3] allow sites and devices to belong to more than one Group --- .../config/global/db-projections.js | 12 +-- src/device-registry/models/Activity.js | 6 +- src/device-registry/models/Airqloud.js | 6 +- src/device-registry/models/Cohort.js | 18 ++--- src/device-registry/models/Device.js | 76 ++++++++++--------- src/device-registry/models/Grid.js | 8 +- src/device-registry/models/Location.js | 8 ++ src/device-registry/models/Photo.js | 8 +- src/device-registry/models/Site.js | 37 ++++++--- src/device-registry/routes/v2/sites.js | 18 +++++ src/device-registry/utils/generate-filter.js | 24 ++++-- .../validators/device.validators.js | 18 +++++ 12 files changed, 157 insertions(+), 82 deletions(-) diff --git a/src/device-registry/config/global/db-projections.js b/src/device-registry/config/global/db-projections.js index 8282998021..37f14abffc 100644 --- a/src/device-registry/config/global/db-projections.js +++ b/src/device-registry/config/global/db-projections.js @@ -135,7 +135,7 @@ const dbProjections = { lat_long: 1, country: 1, network: 1, - group: 1, + groups: 1, data_provider: 1, district: 1, sub_county: 1, @@ -379,7 +379,7 @@ const dbProjections = { mobility: 1, status: 1, network: 1, - group: 1, + groups: 1, api_code: 1, serial_number: 1, authRequired: 1, @@ -591,7 +591,7 @@ const dbProjections = { shape: 1, createdAt: 1, network: 1, - group: 1, + groups: 1, sites: "$sites", numberOfSites: { $cond: { @@ -675,7 +675,7 @@ const dbProjections = { name: 1, description: 1, cohort_tags: 1, - group: 1, + groups: 1, createdAt: 1, visibility: 1, cohort_codes: 1, @@ -817,7 +817,7 @@ const dbProjections = { airqloud_tags: 1, isCustom: 1, network: 1, - group: 1, + groups: 1, metadata: 1, center_point: 1, sites: "$sites", @@ -1065,7 +1065,7 @@ const dbProjections = { date: 1, description: 1, network: 1, - group: 1, + groups: 1, activityType: 1, maintenanceType: 1, recallType: 1, diff --git a/src/device-registry/models/Activity.js b/src/device-registry/models/Activity.js index 0e5863949e..9cd5ada002 100644 --- a/src/device-registry/models/Activity.js +++ b/src/device-registry/models/Activity.js @@ -40,8 +40,8 @@ const activitySchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, activityType: { type: String, trim: true }, @@ -70,7 +70,7 @@ activitySchema.methods = { _id: this._id, device: this.device, network: this.network, - group: this.group, + groups: this.groups, date: this.date, description: this.description, activityType: this.activityType, diff --git a/src/device-registry/models/Airqloud.js b/src/device-registry/models/Airqloud.js index beacd653db..1be15e8683 100644 --- a/src/device-registry/models/Airqloud.js +++ b/src/device-registry/models/Airqloud.js @@ -113,8 +113,8 @@ const airqloudSchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, airqloud_tags: { @@ -152,7 +152,7 @@ airqloudSchema.methods.toJSON = function() { name: this.name, long_name: this.long_name, network: this.network, - group: this.group, + groups: this.groups, description: this.description, airqloud_tags: this.airqloud_tags, admin_level: this.admin_level, diff --git a/src/device-registry/models/Cohort.js b/src/device-registry/models/Cohort.js index 788811e60a..e5aeb92d20 100644 --- a/src/device-registry/models/Cohort.js +++ b/src/device-registry/models/Cohort.js @@ -17,8 +17,8 @@ const cohortSchema = new Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, name: { @@ -81,7 +81,7 @@ cohortSchema.methods.toJSON = function() { cohort_tags, cohort_codes, network, - group, + groups, visibility, } = this; return { @@ -92,7 +92,7 @@ cohortSchema.methods.toJSON = function() { cohort_tags, cohort_codes, network, - group, + groups, }; }; @@ -202,7 +202,7 @@ cohortSchema.statics.list = async function( name: 1, createdAt: 1, network: 1, - group: 1, + groups: 1, devices: { $cond: { if: { $eq: [{ $size: "$devices" }, 0] }, @@ -214,7 +214,7 @@ cohortSchema.statics.list = async function( .sort({ createdAt: -1 }) .project(inclusionProjection) .project(exclusionProjection) - .group({ + .groups({ _id: "$_id", visibility: { $first: "$visibility" }, cohort_tags: { $first: "$cohort_tags" }, @@ -222,7 +222,7 @@ cohortSchema.statics.list = async function( name: { $first: "$name" }, createdAt: { $first: "$createdAt" }, network: { $first: "$network" }, - group: { $first: "$group" }, + groups: { $first: "$groups" }, devices: { $first: "$devices" }, }) .skip(skip ? parseInt(skip) : 0) @@ -240,7 +240,7 @@ cohortSchema.statics.list = async function( name: cohort.name, network: cohort.network, createdAt: cohort.createdAt, - group: cohort.group, + groups: cohort.groups, numberOfDevices: cohort.devices ? cohort.devices.length : 0, devices: cohort.devices ? cohort.devices @@ -250,7 +250,7 @@ cohortSchema.statics.list = async function( status: device.status, name: device.name, network: device.network, - group: device.group, + groups: device.groups, device_number: device.device_number, description: device.description, long_name: device.long_name, diff --git a/src/device-registry/models/Device.js b/src/device-registry/models/Device.js index a5b10c7500..0f5b08749d 100644 --- a/src/device-registry/models/Device.js +++ b/src/device-registry/models/Device.js @@ -74,8 +74,8 @@ const deviceSchema = new mongoose.Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, serial_number: { @@ -237,6 +237,20 @@ deviceSchema.plugin(uniqueValidator, { message: `{VALUE} must be unique!`, }); +const checkDuplicates = (arr, fieldName) => { + const duplicateValues = arr.filter( + (value, index, self) => self.indexOf(value) !== index + ); + + if (duplicateValues.length > 0) { + return new HttpError( + `Duplicate values found in ${fieldName} array.`, + httpStatus.BAD_REQUEST + ); + } + return null; +}; + deviceSchema.pre( [ "update", @@ -326,18 +340,16 @@ deviceSchema.pre( this.device_codes.push(this.serial_number); } - // Check for duplicate values in cohorts array - const duplicateValues = this.cohorts.filter( - (value, index, self) => self.indexOf(value) !== index - ); + // Check for duplicates in cohorts + const cohortsDuplicateError = checkDuplicates(this.cohorts, "cohorts"); + if (cohortsDuplicateError) { + return next(cohortsDuplicateError); + } - if (duplicateValues.length > 0) { - return next( - new HttpError( - "Duplicate values found in cohorts array.", - httpStatus.BAD_REQUEST - ) - ); + // Check for duplicates in groups + const groupsDuplicateError = checkDuplicates(this.groups, "groups"); + if (groupsDuplicateError) { + return next(groupsDuplicateError); } } @@ -371,28 +383,20 @@ deviceSchema.pre( updateData.access_code = access_code.toUpperCase(); } - // Handle $addToSet for device_codes, previous_sites, and pictures - const addToSetUpdates = {}; - - if (updateData.device_codes) { - addToSetUpdates.device_codes = { $each: updateData.device_codes }; - delete updateData.device_codes; // Remove from main update object - } - - if (updateData.previous_sites) { - addToSetUpdates.previous_sites = { $each: updateData.previous_sites }; - delete updateData.previous_sites; // Remove from main update object - } - - if (updateData.pictures) { - addToSetUpdates.pictures = { $each: updateData.pictures }; - delete updateData.pictures; // Remove from main update object - } - - // If there are any $addToSet updates, merge them into the main update object - if (Object.keys(addToSetUpdates).length > 0) { - updateData.$addToSet = addToSetUpdates; - } + // Handle array fields using $addToSet + const arrayFieldsToAddToSet = [ + "device_codes", + "previous_sites", + "groups", + "pictures", + ]; + arrayFieldsToAddToSet.forEach((field) => { + if (updateData[field]) { + updateData.$addToSet = updateData.$addToSet || {}; + updateData.$addToSet[field] = { $each: updateData[field] }; + delete updateData[field]; + } + }); next(); } catch (error) { @@ -415,7 +419,7 @@ deviceSchema.methods = { alias: this.alias, mobility: this.mobility, network: this.network, - group: this.group, + groups: this.groups, api_code: this.api_code, serial_number: this.serial_number, authRequired: this.authRequired, diff --git a/src/device-registry/models/Grid.js b/src/device-registry/models/Grid.js index 53e8e5c027..231704c5b2 100644 --- a/src/device-registry/models/Grid.js +++ b/src/device-registry/models/Grid.js @@ -43,8 +43,8 @@ const gridSchema = new Schema( trim: true, required: [true, "the network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, geoHash: { @@ -121,7 +121,7 @@ gridSchema.methods.toJSON = function() { name, long_name, network, - group, + groups, visibility, description, grid_tags, @@ -139,7 +139,7 @@ gridSchema.methods.toJSON = function() { description, grid_tags, network, - group, + groups, admin_level, grid_codes, centers, diff --git a/src/device-registry/models/Location.js b/src/device-registry/models/Location.js index b37bd7c349..2b894a812f 100644 --- a/src/device-registry/models/Location.js +++ b/src/device-registry/models/Location.js @@ -84,6 +84,10 @@ const locationSchema = new Schema( type: String, trim: true, }, + groups: { + type: [String], + trim: true, + }, location_tags: { type: Array, default: [], @@ -124,6 +128,7 @@ locationSchema.methods = { isCustom: this.isCustom, location: this.location, network: this.network, + groups: this.groups, metadata: this.metadata, }; }, @@ -190,6 +195,7 @@ locationSchema.statics = { isCustom: 1, metadata: 1, network: 1, + groups: 1, sites: "$sites", }; @@ -200,6 +206,7 @@ locationSchema.statics = { admin_level: 1, description: 1, network: 1, + group: 1, metadata: 1, }; @@ -306,6 +313,7 @@ locationSchema.statics = { description: 1, admin_level: 1, network: 1, + group: 1, isCustom: 1, metadata: 1, }, diff --git a/src/device-registry/models/Photo.js b/src/device-registry/models/Photo.js index 78fb7c6679..385be95009 100644 --- a/src/device-registry/models/Photo.js +++ b/src/device-registry/models/Photo.js @@ -18,8 +18,8 @@ const photoSchema = new Schema( type: String, trim: true, }, - group: { - type: String, + groups: { + type: [String], trim: true, }, device_id: { @@ -83,7 +83,7 @@ photoSchema.methods = { tags: this.tags, name: this.name, network: this.network, - group: this.group, + groups: this.groups, image_url: this.image_url, device_id: this.device_id, site_id: this.site_id, @@ -158,7 +158,7 @@ photoSchema.statics = { description: 1, metadata: 1, network: 1, - group: 1, + groups: 1, }) .skip(skip ? skip : 0) .limit(limit ? limit : 1000) diff --git a/src/device-registry/models/Site.js b/src/device-registry/models/Site.js index 101b5fa1a0..3e0946db40 100644 --- a/src/device-registry/models/Site.js +++ b/src/device-registry/models/Site.js @@ -75,8 +75,8 @@ const siteSchema = new Schema( trim: true, required: [true, "network is required!"], }, - group: { - type: String, + groups: { + type: [String], trim: true, }, data_provider: { @@ -364,6 +364,20 @@ const siteSchema = new Schema( } ); +const checkDuplicates = (arr, fieldName) => { + const duplicateValues = arr.filter( + (value, index, self) => self.indexOf(value) !== index + ); + + if (duplicateValues.length > 0) { + return new HttpError( + `Duplicate values found in ${fieldName} array.`, + httpStatus.BAD_REQUEST + ); + } + return null; +}; + siteSchema.pre( ["updateOne", "findOneAndUpdate", "updateMany", "update", "save"], function(next) { @@ -411,6 +425,7 @@ siteSchema.pre( "land_use", "site_codes", "airqlouds", + "groups", "grids", ]; arrayFieldsToAddToSet.forEach((field) => { @@ -442,12 +457,16 @@ siteSchema.pre( if (this[field]) this.site_codes.push(this[field]); }); - // Check for duplicate grid values - const duplicateValues = this.grids.filter( - (value, index, self) => self.indexOf(value) !== index - ); - if (duplicateValues.length > 0) { - return next(new Error("Duplicate values found in grids array.")); + // Check for duplicates in grids + const gridsDuplicateError = checkDuplicates(this.grids, "grids"); + if (gridsDuplicateError) { + return next(gridsDuplicateError); + } + + // Check for duplicates in groups + const groupsDuplicateError = checkDuplicates(this.groups, "groups"); + if (groupsDuplicateError) { + return next(groupsDuplicateError); } } @@ -473,7 +492,7 @@ siteSchema.methods = { generated_name: this.generated_name, search_name: this.search_name, network: this.network, - group: this.group, + groups: this.groups, data_provider: this.data_provider, location_name: this.location_name, formatted_name: this.formatted_name, diff --git a/src/device-registry/routes/v2/sites.js b/src/device-registry/routes/v2/sites.js index 8d3198e89e..8117e500d0 100644 --- a/src/device-registry/routes/v2/sites.js +++ b/src/device-registry/routes/v2/sites.js @@ -486,6 +486,15 @@ router.post( .bail() .notEmpty() .withMessage("the site_tags should not be empty"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("airqlouds") .optional() .custom((value) => { @@ -860,6 +869,15 @@ router.put( .withMessage( "Invalid site_category format, crosscheck the types or content of all the provided nested fields. latitude, longitude & search_radius should be numbers. tags should be an array of strings. category, search_tags & search_radius are required fields" ), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), ], ]), siteController.update diff --git a/src/device-registry/utils/generate-filter.js b/src/device-registry/utils/generate-filter.js index e6320d57b3..e8366e409f 100644 --- a/src/device-registry/utils/generate-filter.js +++ b/src/device-registry/utils/generate-filter.js @@ -1116,7 +1116,7 @@ const generateFilter = { // } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1238,7 +1238,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1370,7 +1370,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1437,7 +1437,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1500,7 +1500,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1597,7 +1597,7 @@ const generateFilter = { } }, locations: (req, next) => { - let { id, name, admin_level, summary, network } = { + let { id, name, admin_level, summary, network, group } = { ...req.query, ...req.params, }; @@ -1623,6 +1623,14 @@ const generateFilter = { ); } + if (group) { + filter.groups = handlePredefinedValueMatch( + group, + constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, + { matchCombinations: true } + ); + } + if (admin_level) { filter["admin_level"] = admin_level; } @@ -1671,7 +1679,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } @@ -1759,7 +1767,7 @@ const generateFilter = { } if (group) { - filter.group = handlePredefinedValueMatch( + filter.groups = handlePredefinedValueMatch( group, constants.PREDEFINED_FILTER_VALUES.COMBINATIONS.GROUP_PAIRS, { matchCombinations: true } diff --git a/src/device-registry/validators/device.validators.js b/src/device-registry/validators/device.validators.js index e0b4916f55..b08baea97d 100644 --- a/src/device-registry/validators/device.validators.js +++ b/src/device-registry/validators/device.validators.js @@ -126,6 +126,15 @@ const validateCreateDevice = [ .isInt() .withMessage("the generation should be an integer") .toInt(), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("mountType") .optional() .notEmpty() @@ -384,6 +393,15 @@ const validateUpdateDevice = [ .trim() .isBoolean() .withMessage("isActive must be Boolean"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("isRetired") .optional() .notEmpty() From 2594d9129d10b6aacb6ad3711beb52aa15441f29 Mon Sep 17 00:00:00 2001 From: baalmart Date: Tue, 10 Dec 2024 18:14:54 +0300 Subject: [PATCH 2/3] input validators for groups and cohorts --- src/device-registry/routes/v2/cohorts.js | 18 ++++++++++++++++++ src/device-registry/routes/v2/grids.js | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/device-registry/routes/v2/cohorts.js b/src/device-registry/routes/v2/cohorts.js index e2d5a07a2b..8f047c0054 100644 --- a/src/device-registry/routes/v2/cohorts.js +++ b/src/device-registry/routes/v2/cohorts.js @@ -128,6 +128,15 @@ router.put( .trim() .isBoolean() .withMessage("visibility must be Boolean"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .optional() .notEmpty() @@ -169,6 +178,15 @@ router.post( .trim() .optional() .notEmpty(), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .trim() .exists() diff --git a/src/device-registry/routes/v2/grids.js b/src/device-registry/routes/v2/grids.js index f70426a4b6..a8678a7a8e 100644 --- a/src/device-registry/routes/v2/grids.js +++ b/src/device-registry/routes/v2/grids.js @@ -195,6 +195,15 @@ router.post( .withMessage( "admin_level values include but not limited to: province, state, village, county, etc. Update your GLOBAL configs" ), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .trim() .optional() @@ -371,6 +380,15 @@ router.put( .optional() .notEmpty() .withMessage("the description should not be empty if provided"), + body("groups") + .optional() + .custom((value) => { + return Array.isArray(value); + }) + .withMessage("the groups should be an array") + .bail() + .notEmpty() + .withMessage("the groups should not be empty"), body("network") .optional() .notEmpty() From 68a0b0f4f5f11f08347f430c1aaf612e3c42e010 Mon Sep 17 00:00:00 2001 From: baalmart Date: Tue, 10 Dec 2024 18:16:07 +0300 Subject: [PATCH 3/3] Fix inconsistent field name in projectSummary --- src/device-registry/models/Location.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/device-registry/models/Location.js b/src/device-registry/models/Location.js index 2b894a812f..b5867155eb 100644 --- a/src/device-registry/models/Location.js +++ b/src/device-registry/models/Location.js @@ -206,7 +206,7 @@ locationSchema.statics = { admin_level: 1, description: 1, network: 1, - group: 1, + groups: 1, metadata: 1, }; @@ -313,7 +313,7 @@ locationSchema.statics = { description: 1, admin_level: 1, network: 1, - group: 1, + groups: 1, isCustom: 1, metadata: 1, },