From 86d03496e7d4475cae070cb5d28b18b824c59c21 Mon Sep 17 00:00:00 2001 From: baalmart Date: Sat, 14 Dec 2024 10:30:07 +0300 Subject: [PATCH 1/3] ix: update preference job to handle compound unique key with group_id --- .../bin/jobs/preferences-update-job.js | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/auth-service/bin/jobs/preferences-update-job.js b/src/auth-service/bin/jobs/preferences-update-job.js index 5a98f6642c..8131b1ccef 100644 --- a/src/auth-service/bin/jobs/preferences-update-job.js +++ b/src/auth-service/bin/jobs/preferences-update-job.js @@ -1,5 +1,6 @@ const cron = require("node-cron"); const UserModel = require("@models/User"); +const mongoose = require("mongoose"); const PreferenceModel = require("@models/Preference"); const SelectedSiteModel = require("@models/SelectedSite"); const constants = require("@config/constants"); @@ -74,6 +75,9 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { return; } + // Specify the group_id you want to use + const defaultGroupId = mongoose.Types.ObjectId("64f54e4621d9b90013925a08"); + while (true) { const users = await UserModel("airqo") .find() @@ -89,7 +93,10 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { // Fetch existing preferences for users in batch const userIds = users.map((user) => user._id); const preferences = await PreferenceModel("airqo") - .find({ user_id: { $in: userIds } }) + .find({ + user_id: { $in: userIds }, + group_id: defaultGroupId, + }) .select("_id user_id selected_sites") .lean(); @@ -103,14 +110,18 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { const userIdStr = user._id.toString(); const preference = preferencesMap.get(userIdStr); + // Prepare the default preference object with the specific group_id + const defaultPreferenceWithGroupId = { + ...defaultPreference, + user_id: user._id, + group_id: defaultGroupId, + selected_sites: selectedSites, + }; + if (!preference) { // No preference exists, create a new one await PreferenceModel("airqo") - .create({ - ...defaultPreference, - user_id: user._id, - selected_sites: selectedSites, - }) + .create(defaultPreferenceWithGroupId) .catch((error) => { logger.error( `🐛🐛 Failed to create preference for user ${userIdStr}: ${stringify( @@ -122,14 +133,18 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { // Preference exists but selected_sites is empty, update it await PreferenceModel("airqo") .findOneAndUpdate( - { _id: preference._id }, { - $set: { - ...defaultPreference, - selected_sites: selectedSites, - }, + user_id: user._id, + group_id: defaultGroupId, }, - { new: true } + { + $set: defaultPreferenceWithGroupId, + }, + { + new: true, + upsert: true, // Add this to handle cases where the document might not exist + setDefaultsOnInsert: true, // Ensures default values are applied when upserting + } ) .catch((error) => { logger.error( From 7170b4a61d72bab0a01e7b9ad37141ed63e0a3d6 Mon Sep 17 00:00:00 2001 From: baalmart Date: Sat, 14 Dec 2024 12:04:26 +0300 Subject: [PATCH 2/3] managing default groups --- .../bin/jobs/preferences-update-job.js | 80 +++++++++++++++---- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/src/auth-service/bin/jobs/preferences-update-job.js b/src/auth-service/bin/jobs/preferences-update-job.js index 8131b1ccef..d07742f54b 100644 --- a/src/auth-service/bin/jobs/preferences-update-job.js +++ b/src/auth-service/bin/jobs/preferences-update-job.js @@ -13,6 +13,30 @@ const stringify = require("@utils/stringify"); const isEmpty = require("is-empty"); const BATCH_SIZE = 100; +// Function to validate critical default values +const validateDefaultValues = () => { + const criticalDefaults = [ + { key: "DEFAULT_GROUP", value: constants.DEFAULT_GROUP }, + { key: "DEFAULT_AIRQLOUD", value: constants.DEFAULT_AIRQLOUD }, + { key: "DEFAULT_GRID", value: constants.DEFAULT_GRID }, + { key: "DEFAULT_NETWORK", value: constants.DEFAULT_NETWORK }, + ]; + + const missingDefaults = criticalDefaults.filter( + (item) => isEmpty(item.value) || item.value === undefined + ); + + if (missingDefaults.length > 0) { + const missingKeys = missingDefaults.map((item) => item.key).join(", "); + logger.error( + `🚨 Aborting preference update: Missing critical default values: ${missingKeys}` + ); + return false; + } + + return true; +}; + // Default preference object const defaultPreference = { pollutant: "pm2_5", @@ -28,10 +52,22 @@ const defaultPreference = { unitValue: 14, unit: "day", }, - airqloud_id: constants.DEFAULT_AIRQLOUD || "NA", - grid_id: constants.DEFAULT_GRID || "NA", - network_id: constants.DEFAULT_NETWORK || "NA", - group_id: constants.DEFAULT_GROUP || "NA", + airqloud_id: constants.DEFAULT_AIRQLOUD, + grid_id: constants.DEFAULT_GRID, + network_id: constants.DEFAULT_NETWORK, + group_id: constants.DEFAULT_GROUP, +}; + +// Function to validate user's group membership +const validateUserGroupMembership = (user, defaultGroupId) => { + // Check if user has group_roles and is a member of the default group + if (!user.group_roles || user.group_roles.length === 0) { + return false; + } + + return user.group_roles.some( + (role) => role.group.toString() === defaultGroupId.toString() + ); }; // Function to get selected sites based on the specified method @@ -63,6 +99,11 @@ const getSelectedSites = async (method = "featured") => { }; const updatePreferences = async (siteSelectionMethod = "featured") => { + // Validate default values before proceeding + if (!validateDefaultValues()) { + return; + } + try { const batchSize = BATCH_SIZE; let skip = 0; @@ -75,38 +116,45 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { return; } - // Specify the group_id you want to use - const defaultGroupId = mongoose.Types.ObjectId("64f54e4621d9b90013925a08"); + // Use constants.DEFAULT_GROUP directly + const defaultGroupId = mongoose.Types.ObjectId(constants.DEFAULT_GROUP); while (true) { + // Fetch users with their group_roles const users = await UserModel("airqo") .find() .limit(batchSize) .skip(skip) - .select("_id") + .select("_id group_roles") .lean(); if (users.length === 0) { break; } - // Fetch existing preferences for users in batch - const userIds = users.map((user) => user._id); + // Filter users who are members of the default group + const validUsers = users.filter((user) => + validateUserGroupMembership(user, defaultGroupId) + ); + + // Get user IDs of valid users + const validUserIds = validUsers.map((user) => user._id); + + // Fetch existing preferences for valid users const preferences = await PreferenceModel("airqo") .find({ - user_id: { $in: userIds }, + user_id: { $in: validUserIds }, group_id: defaultGroupId, }) .select("_id user_id selected_sites") .lean(); const preferencesMap = new Map(); - preferences.forEach((pref) => { preferencesMap.set(pref.user_id.toString(), pref); }); - for (const user of users) { + for (const user of validUsers) { const userIdStr = user._id.toString(); const preference = preferencesMap.get(userIdStr); @@ -119,7 +167,7 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { }; if (!preference) { - // No preference exists, create a new one + // No preference exists for the user in the default group, create a new one await PreferenceModel("airqo") .create(defaultPreferenceWithGroupId) .catch((error) => { @@ -142,8 +190,8 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { }, { new: true, - upsert: true, // Add this to handle cases where the document might not exist - setDefaultsOnInsert: true, // Ensures default values are applied when upserting + upsert: true, + setDefaultsOnInsert: true, } ) .catch((error) => { @@ -169,3 +217,5 @@ cron.schedule(schedule, () => updatePreferences("featured"), { scheduled: true, timezone: "Africa/Nairobi", }); + +module.exports = { updatePreferences }; From 8d97388a55ad3398f6dc4dc52fb4d55759ce0959 Mon Sep 17 00:00:00 2001 From: baalmart Date: Sat, 14 Dec 2024 12:11:27 +0300 Subject: [PATCH 3/3] To avoid data loss, update only the necessary fields --- src/auth-service/bin/jobs/preferences-update-job.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/auth-service/bin/jobs/preferences-update-job.js b/src/auth-service/bin/jobs/preferences-update-job.js index d07742f54b..53eb337a16 100644 --- a/src/auth-service/bin/jobs/preferences-update-job.js +++ b/src/auth-service/bin/jobs/preferences-update-job.js @@ -186,7 +186,10 @@ const updatePreferences = async (siteSelectionMethod = "featured") => { group_id: defaultGroupId, }, { - $set: defaultPreferenceWithGroupId, + $set: { + selected_sites: selectedSites, + group_id: defaultGroupId, + }, }, { new: true,