Merge pull request #3593 from airqo-platform/adding-more-logs

just adding more logs for failing events storage
airqo-platform · Oct 8, 2024 · e9832ae · e9832ae
2 parents cd41905 + 5219e28
commit e9832ae
Show file tree

Hide file tree

Showing 9 changed files with 82 additions and 77 deletions.
diff --git a/src/device-registry/bin/index.js b/src/device-registry/bin/index.js
@@ -9,7 +9,7 @@ const constants = require("@config/constants");
 const log4jsConfiguration = require("@config/log4js");
 log4js.configure(log4jsConfiguration);
 const logger = log4js.getLogger(`${constants.ENVIRONMENT} -- bin/index`);
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 
 try {
   require("fs").mkdirSync("./log");
@@ -24,7 +24,7 @@ const startKafka = async () => {
   await kafkaConsumer().catch((error) => {
     logObject("KAFKA error in the main()", error);
     logger.error(
-      `🐛🐛 KAFKA: internal server error in the main() -- ${jsonify(error)}`
+      `🐛🐛 KAFKA: internal server error in the main() -- ${stringify(error)}`
     );
     logger.error(
       `🐛🐛 KAFKA error message: internal server error in the main() -- ${error.message}`
@@ -37,11 +37,11 @@ const main = async () => {
     await startKafka();
     createServer();
   } catch (error) {
-    logger.error(`🐛🐛 error in the main() -- ${jsonify(error)}`);
+    logger.error(`🐛🐛 error in the main() -- ${stringify(error)}`);
   }
 };
 
 main().catch((error) => {
   console.error("🐛🐛 Error starting the application: ", error);
-  logger.error(`🐛🐛 Error starting the application -- ${jsonify(error)}`);
+  logger.error(`🐛🐛 Error starting the application -- ${stringify(error)}`);
 });
diff --git a/src/device-registry/bin/jobs/kafka-consumer.js b/src/device-registry/bin/jobs/kafka-consumer.js
@@ -10,7 +10,7 @@ const Joi = require("joi");
 const { jsonrepair } = require("jsonrepair");
 const cleanDeep = require("clean-deep");
 const isEmpty = require("is-empty");
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 
 const eventSchema = Joi.object({
   s2_pm2_5: Joi.number().optional(),
@@ -68,7 +68,7 @@ const consumeHourlyMeasurements = async (messageData) => {
   try {
     if (isEmpty(messageData)) {
       logger.error(
-        `KAFKA: the sent message in undefined --- ${jsonify(messageData)}`
+        `KAFKA: the sent message in undefined --- ${stringify(messageData)}`
       );
     }
     const repairedJSONString = jsonrepair(messageData);
@@ -78,7 +78,7 @@ const consumeHourlyMeasurements = async (messageData) => {
     // const measurements = JSON.parse(repairedJSONString);
     if (!Array.isArray(measurements) || isEmpty(measurements)) {
       // logger.error(
-      //   `KAFKA: the sent measurements are not an array or they are just empty (undefined) --- ${jsonify(
+      //   `KAFKA: the sent measurements are not an array or they are just empty (undefined) --- ${stringify(
       //     measurements
       //   )}`
       // );
@@ -122,17 +122,17 @@ const consumeHourlyMeasurements = async (messageData) => {
           // };
         });
         // logger.error(
-        //   `KAFKA: Input validation formatted errors -- ${jsonify(
+        //   `KAFKA: Input validation formatted errors -- ${stringify(
         //     errorDetails
         //   )}`
         // );
 
         // logger.error(
-        //     `KAFKA: ALL the input validation errors --- ${jsonify(error.details)}`
+        //     `KAFKA: ALL the input validation errors --- ${stringify(error.details)}`
         // );
 
         // logger.info(
-        //     `KAFKA: the VALUE for ALL the shared input validation errors --- ${jsonify(value)}`
+        //     `KAFKA: the VALUE for ALL the shared input validation errors --- ${stringify(value)}`
         // );
       }
       // logObject("value", value);
@@ -150,14 +150,14 @@ const consumeHourlyMeasurements = async (messageData) => {
       if (responseFromInsertMeasurements.success === false) {
         console.log("KAFKA: failed to store the measurements");
         // logger.error(
-        //   `KAFKA: responseFromInsertMeasurements --- ${jsonify(
+        //   `KAFKA: responseFromInsertMeasurements --- ${stringify(
         //     responseFromInsertMeasurements
         //   )}`
         // );
       } else if (responseFromInsertMeasurements.success === true) {
         console.log("KAFKA: successfully stored the measurements");
         // logger.info(
-        //     `KAFKA: successfully inserted the measurements --- ${jsonify(responseFromInsertMeasurements.message ?
+        //     `KAFKA: successfully inserted the measurements --- ${stringify(responseFromInsertMeasurements.message ?
         //     responseFromInsertMeasurements.message :
         //     "")}`
         // );
@@ -172,7 +172,7 @@ const consumeHourlyMeasurements = async (messageData) => {
       `ℹ️ℹ️ incoming KAFKA value's TYPE which is causing errors --- ${typeof message.value}`
     );
     logger.error(`🐛🐛 KAFKA: error message --- ${error.message}`);
-    logger.error(`🐛🐛 KAFKA: full error object --- ${jsonify(error)}`);
+    logger.error(`🐛🐛 KAFKA: full error object --- ${stringify(error)}`);
   }
 };
 
@@ -212,7 +212,7 @@ const kafkaConsumer = async () => {
               }
             } catch (error) {
               logger.error(
-                `🐛🐛 Error processing Kafka message for topic ${topic}: ${jsonify(
+                `🐛🐛 Error processing Kafka message for topic ${topic}: ${stringify(
                   error
                 )}`
               );
@@ -223,7 +223,7 @@ const kafkaConsumer = async () => {
     );
   } catch (error) {
     logObject("Error connecting to Kafka", error);
-    logger.error(`📶📶 Error connecting to Kafka: ${jsonify(error)}`);
+    logger.error(`📶📶 Error connecting to Kafka: ${stringify(error)}`);
   }
 };
 

diff --git a/src/device-registry/bin/jobs/new-store-readings-job.js b/src/device-registry/bin/jobs/new-store-readings-job.js
@@ -8,7 +8,7 @@ const DeviceModel = require("@models/Device");
 const SiteModel = require("@models/Site");
 const ReadingModel = require("@models/Reading");
 const { logText, logObject } = require("@utils/log");
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 const asyncRetry = require("async-retry");
 const generateFilter = require("@utils/generate-filter");
 const cron = require("node-cron");
@@ -55,7 +55,7 @@ async function updateEntityStatus(Model, filter, time, entityType) {
       };
       const updateResult = await Model.updateOne(filter, updateData);
     } else {
-      logger.warn(`${entityType} not found with filter: ${jsonify(filter)}`);
+      logger.warn(`${entityType} not found with filter: ${stringify(filter)}`);
     }
   } catch (error) {
     logger.error(`Error updating ${entityType}'s status: ${error.message}`);
@@ -129,13 +129,13 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
       viewEventsResponse = await EventModel("airqo").fetch(filter);
       logText("Running the data insertion script");
     } catch (fetchError) {
-      logger.error(`Error fetching events: ${jsonify(fetchError)}`);
+      logger.error(`Error fetching events: ${stringify(fetchError)}`);
       return;
     }
 
     if (!viewEventsResponse || typeof viewEventsResponse !== "object") {
       logger.error(
-        `Unexpected response from EventModel.fetch(): ${jsonify(
+        `Unexpected response from EventModel.fetch(): ${stringify(
           viewEventsResponse
         )}`
       );
@@ -175,15 +175,15 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
                   logObject("the error inside processing of batches", error);
                   if (error.name === "MongoError" && error.code !== 11000) {
                     logger.error(
-                      `🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${jsonify(
+                      `🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${stringify(
                         error
                       )}`
                     );
                     throw error; // Retry the operation
                   } else if (error.code === 11000) {
                     // Ignore duplicate key errors
                     console.warn(
-                      `Duplicate key error for document: ${jsonify(doc)}`
+                      `Duplicate key error for document: ${stringify(doc)}`
                     );
                   }
                 }
@@ -204,15 +204,15 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
         viewEventsResponse
       );
       logger.error(
-        `🐛🐛 Unable to retrieve Events to insert into Readings -- ${jsonify(
+        `🐛🐛 Unable to retrieve Events to insert into Readings -- ${stringify(
           viewEventsResponse
         )}`
       );
       logText(`🐛🐛 Unable to retrieve Events to insert into Readings`);
     }
   } catch (error) {
     logObject("error", error);
-    logger.error(`🐛🐛 Internal Server Error ${jsonify(error)}`);
+    logger.error(`🐛🐛 Internal Server Error ${stringify(error)}`);
   }
 };
 

diff --git a/src/device-registry/bin/jobs/store-readings-job.js b/src/device-registry/bin/jobs/store-readings-job.js
@@ -6,7 +6,7 @@ const logger = log4js.getLogger(
 const EventModel = require("@models/Event");
 const ReadingModel = require("@models/Reading");
 const { logText, logObject } = require("@utils/log");
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 const asyncRetry = require("async-retry");
 const generateFilter = require("@utils/generate-filter");
 const cron = require("node-cron");
@@ -67,15 +67,15 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
               } catch (error) {
                 if (error.name === "MongoError" && error.code !== 11000) {
                   logger.error(
-                    `🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${jsonify(
+                    `🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${stringify(
                       error
                     )}`
                   );
                   throw error; // Retry the operation
                 } else if (error.code === 11000) {
                   // Ignore duplicate key errors
                   console.warn(
-                    `Duplicate key error for document: ${jsonify(doc)}`
+                    `Duplicate key error for document: ${stringify(doc)}`
                   );
                 }
               }
@@ -97,7 +97,7 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
       );
 
       logger.error(
-        `🐛🐛 Unable to retrieve Events to insert into Readings -- ${jsonify(
+        `🐛🐛 Unable to retrieve Events to insert into Readings -- ${stringify(
           viewEventsResponse
         )}`
       );

diff --git a/src/device-registry/bin/jobs/store-signals-job.js b/src/device-registry/bin/jobs/store-signals-job.js
@@ -6,7 +6,7 @@ const logger = log4js.getLogger(
 const EventModel = require("@models/Event");
 const SignalModel = require("@models/Signal");
 const { logText, logObject } = require("@utils/log");
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 const asyncRetry = require("async-retry");
 const generateFilter = require("@utils/generate-filter");
 const cron = require("node-cron");
@@ -67,15 +67,15 @@ const fetchAndStoreDataIntoSignalsModel = async () => {
               } catch (error) {
                 if (error.name === "MongoError" && error.code !== 11000) {
                   logger.error(
-                    `🐛🐛 MongoError -- fetchAndStoreDataIntoSignalsModel -- ${jsonify(
+                    `🐛🐛 MongoError -- fetchAndStoreDataIntoSignalsModel -- ${stringify(
                       error
                     )}`
                   );
                   throw error; // Retry the operation
                 } else if (error.code === 11000) {
                   // Ignore duplicate key errors
                   console.warn(
-                    `Duplicate key error for document: ${jsonify(doc)}`
+                    `Duplicate key error for document: ${stringify(doc)}`
                   );
                 }
               }
@@ -97,7 +97,7 @@ const fetchAndStoreDataIntoSignalsModel = async () => {
       );
 
       logger.error(
-        `🐛🐛 Unable to retrieve Events to insert into Signals -- ${jsonify(
+        `🐛🐛 Unable to retrieve Events to insert into Signals -- ${stringify(
           viewEventsResponse
         )}`
       );

diff --git a/src/device-registry/bin/server.js b/src/device-registry/bin/server.js
@@ -22,7 +22,7 @@ const debug = require("debug")("auth-service:server");
 const isEmpty = require("is-empty");
 const logger = log4js.getLogger(`${constants.ENVIRONMENT} -- bin/server`);
 const { logText, logObject } = require("@utils/log");
-const jsonify = require("@utils/jsonify");
+const stringify = require("@utils/stringify");
 require("@bin/jobs/store-signals-job");
 require("@bin/jobs/new-store-readings-job");
 // require("@bin/jobs/new-check-network-status-job");
@@ -90,28 +90,28 @@ app.use(function(err, req, res, next) {
         errors: { message: err.message },
       });
     } else if (err.status === 400) {
-      logger.error(`Bad request error --- ${jsonify(err)}`);
+      logger.error(`Bad request error --- ${stringify(err)}`);
       res.status(err.status).json({
         success: false,
         message: "Bad request error",
         errors: { message: err.message },
       });
     } else if (err.status === 401) {
-      logger.error(`Unauthorized --- ${jsonify(err)}`);
+      logger.error(`Unauthorized --- ${stringify(err)}`);
       res.status(err.status).json({
         success: false,
         message: "Unauthorized",
         errors: { message: err.message },
       });
     } else if (err.status === 403) {
-      logger.error(`Forbidden --- ${jsonify(err)}`);
+      logger.error(`Forbidden --- ${stringify(err)}`);
       res.status(err.status).json({
         success: false,
         message: "Forbidden",
         errors: { message: err.message },
       });
     } else if (err.status === 500) {
-      // logger.error(`Internal Server Error --- ${jsonify(err)}`);
+      // logger.error(`Internal Server Error --- ${stringify(err)}`);
       // logger.error(`Stack Trace: ${err.stack}`);
       logObject("the error", err);
       res.status(err.status).json({
@@ -120,14 +120,14 @@ app.use(function(err, req, res, next) {
         errors: { message: err.message },
       });
     } else if (err.status === 502 || err.status === 503 || err.status === 504) {
-      logger.error(`${err.message} --- ${jsonify(err)}`);
+      logger.error(`${err.message} --- ${stringify(err)}`);
       res.status(err.status).json({
         success: false,
         message: err.message,
         errors: { message: err.message },
       });
     } else {
-      logger.error(`Internal Server Error --- ${jsonify(err)}`);
+      logger.error(`Internal Server Error --- ${stringify(err)}`);
       logObject("Internal Server Error", err);
       logger.error(`Stack Trace: ${err.stack}`);
       res.status(err.status || 500).json({
@@ -138,7 +138,7 @@ app.use(function(err, req, res, next) {
     }
   } else {
     logger.info(
-      `🍻🍻 HTTP response already sent to the client -- ${jsonify(err)}`
+      `🍻🍻 HTTP response already sent to the client -- ${stringify(err)}`
     );
   }
 });