From d0fafc4adaa2e16ac2d0d20ec09bdbdff811cb57 Mon Sep 17 00:00:00 2001 From: baalmart Date: Fri, 20 Dec 2024 09:12:44 +0300 Subject: [PATCH 1/4] update duplicate field names in Sites collection --- .../jobs/update-duplicate-site-fields-job.js | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/device-registry/bin/jobs/update-duplicate-site-fields-job.js diff --git a/src/device-registry/bin/jobs/update-duplicate-site-fields-job.js b/src/device-registry/bin/jobs/update-duplicate-site-fields-job.js new file mode 100644 index 0000000000..c1d9b9b615 --- /dev/null +++ b/src/device-registry/bin/jobs/update-duplicate-site-fields-job.js @@ -0,0 +1,164 @@ +const constants = require("@config/constants"); +const cron = require("node-cron"); +const log4js = require("log4js"); +const logger = log4js.getLogger( + `${constants.ENVIRONMENT} -- /bin/jobs/update-duplicate-site-fields-job` +); +const SitesModel = require("@models/Site"); +const { logText, logObject } = require("@utils/log"); + +// Fields to check and update duplicates +const FIELDS_TO_UPDATE = ["name", "search_name", "description"]; + +// Frequency configuration +const WARNING_FREQUENCY_HOURS = 4; // Change this value to adjust frequency + +// Helper function to extract site number from generated_name +const extractSiteNumber = (generated_name) => { + const match = generated_name.match(/site_(\d+)/); + return match ? match[1] : null; +}; + +// Helper function to group sites by field value +const groupSitesByFieldValue = (sites, fieldName) => { + return sites.reduce((acc, site) => { + const fieldValue = site[fieldName]; + if (!fieldValue) return acc; + + if (!acc[fieldValue]) { + acc[fieldValue] = []; + } + acc[fieldValue].push(site); + return acc; + }, {}); +}; + +// Function to generate unique field value using site number +const generateUniqueFieldValue = (originalValue, siteNumber) => { + return `${originalValue} ${siteNumber}`; +}; + +// Function to update duplicate fields for a specific field +const updateDuplicatesForField = async (groupedSites, fieldName) => { + const updates = []; + const updatedValues = new Set(); + + for (const [fieldValue, sites] of Object.entries(groupedSites)) { + if (sites.length > 1) { + // Sort sites by generated_name to ensure consistent numbering + sites.sort((a, b) => a.generated_name.localeCompare(b.generated_name)); + + // Update all sites except the first one (keep original for the first occurrence) + for (let i = 1; i < sites.length; i++) { + const site = sites[i]; + const siteNumber = extractSiteNumber(site.generated_name); + + if (siteNumber) { + const newValue = generateUniqueFieldValue(fieldValue, siteNumber); + updates.push({ + updateOne: { + filter: { _id: site._id }, + update: { [fieldName]: newValue }, + }, + }); + updatedValues.add( + `${site.generated_name}: ${fieldValue} -> ${newValue}` + ); + } + } + } + } + + if (updates.length > 0) { + try { + const result = await SitesModel("airqo").bulkWrite(updates); + return { + field: fieldName, + updatedCount: result.modifiedCount, + updates: Array.from(updatedValues), + }; + } catch (error) { + logger.error(`Error updating ${fieldName}: ${error.message}`); + throw error; + } + } + + return null; +}; + +// Main function to update duplicate field values +const updateDuplicateSiteFields = async () => { + try { + logText("Starting duplicate site fields update process..."); + + // Get all active sites + const fieldsToProject = FIELDS_TO_UPDATE.concat([ + "_id", + "generated_name", + ]).join(" "); + + const sites = await SitesModel("airqo").find( + { isOnline: true }, + fieldsToProject + ); + + logObject("Total sites to process", sites.length); + + const updateReport = { + totalUpdates: 0, + fieldReports: [], + }; + + // Process each field + for (const field of FIELDS_TO_UPDATE) { + const groupedSites = groupSitesByFieldValue(sites, field); + const updateResult = await updateDuplicatesForField(groupedSites, field); + + if (updateResult) { + updateReport.totalUpdates += updateResult.updatedCount; + updateReport.fieldReports.push(updateResult); + } + } + + // Log results + if (updateReport.totalUpdates > 0) { + logText("🔄 Site field updates completed!"); + let updateMessage = `Updated ${updateReport.totalUpdates} duplicate values:\n`; + + updateReport.fieldReports.forEach((report) => { + updateMessage += `\nField: ${report.field} (${report.updatedCount} updates)`; + report.updates.forEach((update) => { + updateMessage += `\n - ${update}`; + }); + }); + + logger.info(updateMessage); + logText(updateMessage); + } else { + logText("✅ No duplicate fields requiring updates"); + logger.info("No duplicate fields requiring updates"); + } + } catch (error) { + const errorMessage = `🐛 Error updating duplicate site fields: ${error.message}`; + logText(errorMessage); + logger.error(errorMessage); + logger.error(`Stack trace: ${error.stack}`); + } +}; + +// Initial run message +logText("Update duplicate site fields job is now running....."); +// Schedule the job to run every 4 hours at minute 15 +const schedule = `15 */${WARNING_FREQUENCY_HOURS} * * *`; +cron.schedule(schedule, updateDuplicateSiteFields, { + scheduled: true, +}); + +// Export for testing or manual execution +module.exports = { + updateDuplicateSiteFields, + FIELDS_TO_UPDATE, + // Export helpers for testing + extractSiteNumber, + generateUniqueFieldValue, +}; From 16b3e7197c53d723492b4bc2fe75e0a537852580 Mon Sep 17 00:00:00 2001 From: baalmart Date: Fri, 20 Dec 2024 09:13:01 +0300 Subject: [PATCH 2/4] update duplicate field names in Sites collection --- src/device-registry/bin/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/device-registry/bin/server.js b/src/device-registry/bin/server.js index ec83b750ee..6822e0f473 100644 --- a/src/device-registry/bin/server.js +++ b/src/device-registry/bin/server.js @@ -30,6 +30,7 @@ require("@bin/jobs/check-unassigned-devices-job"); require("@bin/jobs/check-active-statuses"); require("@bin/jobs/check-unassigned-sites-job"); require("@bin/jobs/check-duplicate-site-fields-job"); +require("@bin/jobs/update-duplicate-site-fields-job"); if (isEmpty(constants.SESSION_SECRET)) { throw new Error("SESSION_SECRET environment variable not set"); } From 81777dea6956025bc1c317f64285f7ca61a2d1e6 Mon Sep 17 00:00:00 2001 From: baalmart Date: Fri, 20 Dec 2024 09:13:47 +0300 Subject: [PATCH 3/4] when assigning Group, also update data_provider --- src/device-registry/models/Site.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/device-registry/models/Site.js b/src/device-registry/models/Site.js index 8e58617e34..84aeebcc22 100644 --- a/src/device-registry/models/Site.js +++ b/src/device-registry/models/Site.js @@ -393,6 +393,31 @@ siteSchema.pre( if (this.getUpdate) { const updates = this.getUpdate(); if (updates) { + // Handle data_provider update based on groups + const hasExplicitDataProvider = + updates.data_provider || (updates.$set && updates.$set.data_provider); + + // Check for groups in different possible update operations + const groupsUpdate = + updates.groups || + (updates.$set && updates.$set.groups) || + (updates.$addToSet && + updates.$addToSet.groups && + updates.$addToSet.groups.$each) || + (updates.$push && updates.$push.groups && updates.$push.groups.$each); + + // Update data_provider if groups are being updated and no explicit data_provider is provided + if (groupsUpdate && !hasExplicitDataProvider) { + const groupsArray = Array.isArray(groupsUpdate) + ? groupsUpdate + : groupsUpdate.$each + ? groupsUpdate.$each + : [groupsUpdate]; + + if (groupsArray.length > 0) { + updates.data_provider = groupsArray[0]; // Direct assignment instead of using $set + } + } // Prevent modification of restricted fields const restrictedFields = [ "latitude", From 73bcd492127f86c5081a0ba6e152cf33d0bc901d Mon Sep 17 00:00:00 2001 From: baalmart Date: Fri, 20 Dec 2024 10:19:01 +0300 Subject: [PATCH 4/4] ensure that the category field value is never removed in any type of device update operation --- src/device-registry/models/Device.js | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/device-registry/models/Device.js b/src/device-registry/models/Device.js index 8bf64e09c3..75e936c8bc 100644 --- a/src/device-registry/models/Device.js +++ b/src/device-registry/models/Device.js @@ -24,6 +24,10 @@ const minLength = [ const noSpaces = /^\S*$/; +const DEVICE_CONFIG = { + ALLOWED_CATEGORIES: ["bam", "lowcost", "gas"], +}; + const accessCodeGenerator = require("generate-password"); function sanitizeObject(obj, invalidKeys) { @@ -266,6 +270,45 @@ deviceSchema.pre( const isNew = this.isNew; const updateData = this.getUpdate ? this.getUpdate() : this; + // Handle category field for both new documents and updates + if (isNew) { + // For new documents + if ("category" in this) { + if (this.category === null) { + delete this.category; + } else if ( + !DEVICE_CONFIG.ALLOWED_CATEGORIES.includes(this.category) + ) { + return next( + new HttpError( + `Invalid category. Must be one of: ${DEVICE_CONFIG.ALLOWED_CATEGORIES.join( + ", " + )}`, + httpStatus.BAD_REQUEST + ) + ); + } + } + } else { + // For updates + if ("category" in updateData) { + if (updateData.category === null) { + delete updateData.category; + } else if ( + !DEVICE_CONFIG.ALLOWED_CATEGORIES.includes(updateData.category) + ) { + return next( + new HttpError( + `Invalid category. Must be one of: ${DEVICE_CONFIG.ALLOWED_CATEGORIES.join( + ", " + )}`, + httpStatus.BAD_REQUEST + ) + ); + } + } + } + if (isNew) { // Set default network if not provided if (!this.network) {