Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

just adding more logs for failing events storage #3593

Merged
merged 1 commit into from
Oct 8, 2024
Merged

just adding more logs for failing events storage #3593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/device-registry/bin/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const constants = require("@config/constants");
const log4jsConfiguration = require("@config/log4js");
log4js.configure(log4jsConfiguration);
const logger = log4js.getLogger(`${constants.ENVIRONMENT} -- bin/index`);
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");

try {
require("fs").mkdirSync("./log");
Expand All @@ -24,7 +24,7 @@ const startKafka = async () => {
await kafkaConsumer().catch((error) => {
logObject("KAFKA error in the main()", error);
logger.error(
`🐛🐛 KAFKA: internal server error in the main() -- ${jsonify(error)}`
`🐛🐛 KAFKA: internal server error in the main() -- ${stringify(error)}`
);
logger.error(
`🐛🐛 KAFKA error message: internal server error in the main() -- ${error.message}`
Expand All @@ -37,11 +37,11 @@ const main = async () => {
await startKafka();
createServer();
} catch (error) {
logger.error(`🐛🐛 error in the main() -- ${jsonify(error)}`);
logger.error(`🐛🐛 error in the main() -- ${stringify(error)}`);
}
};

main().catch((error) => {
console.error("🐛🐛 Error starting the application: ", error);
logger.error(`🐛🐛 Error starting the application -- ${jsonify(error)}`);
logger.error(`🐛🐛 Error starting the application -- ${stringify(error)}`);
});
22 changes: 11 additions & 11 deletions src/device-registry/bin/jobs/kafka-consumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
const { jsonrepair } = require("jsonrepair");
const cleanDeep = require("clean-deep");
const isEmpty = require("is-empty");
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");

const eventSchema = Joi.object({
s2_pm2_5: Joi.number().optional(),
Expand Down Expand Up @@ -68,7 +68,7 @@
try {
if (isEmpty(messageData)) {
logger.error(
`KAFKA: the sent message in undefined --- ${jsonify(messageData)}`
`KAFKA: the sent message in undefined --- ${stringify(messageData)}`
);
}
const repairedJSONString = jsonrepair(messageData);
Expand All @@ -78,7 +78,7 @@
// const measurements = JSON.parse(repairedJSONString);
if (!Array.isArray(measurements) || isEmpty(measurements)) {
// logger.error(
// `KAFKA: the sent measurements are not an array or they are just empty (undefined) --- ${jsonify(
// `KAFKA: the sent measurements are not an array or they are just empty (undefined) --- ${stringify(
// measurements
// )}`
// );
Expand Down Expand Up @@ -122,17 +122,17 @@
// };
});
// logger.error(
// `KAFKA: Input validation formatted errors -- ${jsonify(
// `KAFKA: Input validation formatted errors -- ${stringify(
// errorDetails
// )}`
// );

// logger.error(
// `KAFKA: ALL the input validation errors --- ${jsonify(error.details)}`
// `KAFKA: ALL the input validation errors --- ${stringify(error.details)}`
// );

// logger.info(
// `KAFKA: the VALUE for ALL the shared input validation errors --- ${jsonify(value)}`
// `KAFKA: the VALUE for ALL the shared input validation errors --- ${stringify(value)}`
// );
}
// logObject("value", value);
Expand All @@ -150,14 +150,14 @@
if (responseFromInsertMeasurements.success === false) {
console.log("KAFKA: failed to store the measurements");
// logger.error(
// `KAFKA: responseFromInsertMeasurements --- ${jsonify(
// `KAFKA: responseFromInsertMeasurements --- ${stringify(
// responseFromInsertMeasurements
// )}`
// );
} else if (responseFromInsertMeasurements.success === true) {
console.log("KAFKA: successfully stored the measurements");
// logger.info(
// `KAFKA: successfully inserted the measurements --- ${jsonify(responseFromInsertMeasurements.message ?
// `KAFKA: successfully inserted the measurements --- ${stringify(responseFromInsertMeasurements.message ?
// responseFromInsertMeasurements.message :
// "")}`
// );
Expand All @@ -172,7 +172,7 @@
`ℹ️ℹ️ incoming KAFKA value's TYPE which is causing errors --- ${typeof message.value}`
);
logger.error(`🐛🐛 KAFKA: error message --- ${error.message}`);
logger.error(`🐛🐛 KAFKA: full error object --- ${jsonify(error)}`);
logger.error(`🐛🐛 KAFKA: full error object --- ${stringify(error)}`);

Check warning on line 175 in src/device-registry/bin/jobs/kafka-consumer.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/jobs/kafka-consumer.js#L175 

Added line #L175 was not covered by tests
}
};

Expand Down Expand Up @@ -212,7 +212,7 @@
}
} catch (error) {
logger.error(
`🐛🐛 Error processing Kafka message for topic ${topic}: ${jsonify(
`🐛🐛 Error processing Kafka message for topic ${topic}: ${stringify(
error
)}`
);
Expand All @@ -223,7 +223,7 @@
);
} catch (error) {
logObject("Error connecting to Kafka", error);
logger.error(`📶📶 Error connecting to Kafka: ${jsonify(error)}`);
logger.error(`📶📶 Error connecting to Kafka: ${stringify(error)}`);

Check warning on line 226 in src/device-registry/bin/jobs/kafka-consumer.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/jobs/kafka-consumer.js#L226 

Added line #L226 was not covered by tests
}
};

Expand Down
16 changes: 8 additions & 8 deletions src/device-registry/bin/jobs/new-store-readings-job.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
const SiteModel = require("@models/Site");
const ReadingModel = require("@models/Reading");
const { logText, logObject } = require("@utils/log");
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");
const asyncRetry = require("async-retry");
const generateFilter = require("@utils/generate-filter");
const cron = require("node-cron");
Expand Down Expand Up @@ -55,7 +55,7 @@
};
const updateResult = await Model.updateOne(filter, updateData);
} else {
logger.warn(`${entityType} not found with filter: ${jsonify(filter)}`);
logger.warn(`${entityType} not found with filter: ${stringify(filter)}`);

Check warning on line 58 in src/device-registry/bin/jobs/new-store-readings-job.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/jobs/new-store-readings-job.js#L58 

Added line #L58 was not covered by tests
}
} catch (error) {
logger.error(`Error updating ${entityType}'s status: ${error.message}`);
Expand Down Expand Up @@ -129,13 +129,13 @@
viewEventsResponse = await EventModel("airqo").fetch(filter);
logText("Running the data insertion script");
} catch (fetchError) {
logger.error(`Error fetching events: ${jsonify(fetchError)}`);
logger.error(`Error fetching events: ${stringify(fetchError)}`);

Check warning on line 132 in src/device-registry/bin/jobs/new-store-readings-job.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/jobs/new-store-readings-job.js#L132 

Added line #L132 was not covered by tests
return;
}

if (!viewEventsResponse || typeof viewEventsResponse !== "object") {
logger.error(
`Unexpected response from EventModel.fetch(): ${jsonify(
`Unexpected response from EventModel.fetch(): ${stringify(
viewEventsResponse
)}`
);
Expand Down Expand Up @@ -175,15 +175,15 @@
logObject("the error inside processing of batches", error);
if (error.name === "MongoError" && error.code !== 11000) {
logger.error(
`🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${jsonify(
`🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${stringify(
error
)}`
);
throw error; // Retry the operation
} else if (error.code === 11000) {
// Ignore duplicate key errors
console.warn(
`Duplicate key error for document: ${jsonify(doc)}`
`Duplicate key error for document: ${stringify(doc)}`
);
}
}
Expand All @@ -204,15 +204,15 @@
viewEventsResponse
);
logger.error(
`🐛🐛 Unable to retrieve Events to insert into Readings -- ${jsonify(
`🐛🐛 Unable to retrieve Events to insert into Readings -- ${stringify(
viewEventsResponse
)}`
);
logText(`🐛🐛 Unable to retrieve Events to insert into Readings`);
}
} catch (error) {
logObject("error", error);
logger.error(`🐛🐛 Internal Server Error ${jsonify(error)}`);
logger.error(`🐛🐛 Internal Server Error ${stringify(error)}`);

Check warning on line 215 in src/device-registry/bin/jobs/new-store-readings-job.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/jobs/new-store-readings-job.js#L215 

Added line #L215 was not covered by tests
}
};

Expand Down
8 changes: 4 additions & 4 deletions src/device-registry/bin/jobs/store-readings-job.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const logger = log4js.getLogger(
const EventModel = require("@models/Event");
const ReadingModel = require("@models/Reading");
const { logText, logObject } = require("@utils/log");
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");
const asyncRetry = require("async-retry");
const generateFilter = require("@utils/generate-filter");
const cron = require("node-cron");
Expand Down Expand Up @@ -67,15 +67,15 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
} catch (error) {
if (error.name === "MongoError" && error.code !== 11000) {
logger.error(
`🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${jsonify(
`🐛🐛 MongoError -- fetchAndStoreDataIntoReadingsModel -- ${stringify(
error
)}`
);
throw error; // Retry the operation
} else if (error.code === 11000) {
// Ignore duplicate key errors
console.warn(
`Duplicate key error for document: ${jsonify(doc)}`
`Duplicate key error for document: ${stringify(doc)}`
);
}
}
Expand All @@ -97,7 +97,7 @@ const fetchAndStoreDataIntoReadingsModel = async () => {
);

logger.error(
`🐛🐛 Unable to retrieve Events to insert into Readings -- ${jsonify(
`🐛🐛 Unable to retrieve Events to insert into Readings -- ${stringify(
viewEventsResponse
)}`
);
Expand Down
8 changes: 4 additions & 4 deletions src/device-registry/bin/jobs/store-signals-job.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const logger = log4js.getLogger(
const EventModel = require("@models/Event");
const SignalModel = require("@models/Signal");
const { logText, logObject } = require("@utils/log");
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");
const asyncRetry = require("async-retry");
const generateFilter = require("@utils/generate-filter");
const cron = require("node-cron");
Expand Down Expand Up @@ -67,15 +67,15 @@ const fetchAndStoreDataIntoSignalsModel = async () => {
} catch (error) {
if (error.name === "MongoError" && error.code !== 11000) {
logger.error(
`🐛🐛 MongoError -- fetchAndStoreDataIntoSignalsModel -- ${jsonify(
`🐛🐛 MongoError -- fetchAndStoreDataIntoSignalsModel -- ${stringify(
error
)}`
);
throw error; // Retry the operation
} else if (error.code === 11000) {
// Ignore duplicate key errors
console.warn(
`Duplicate key error for document: ${jsonify(doc)}`
`Duplicate key error for document: ${stringify(doc)}`
);
}
}
Expand All @@ -97,7 +97,7 @@ const fetchAndStoreDataIntoSignalsModel = async () => {
);

logger.error(
`🐛🐛 Unable to retrieve Events to insert into Signals -- ${jsonify(
`🐛🐛 Unable to retrieve Events to insert into Signals -- ${stringify(
viewEventsResponse
)}`
);
Expand Down
16 changes: 8 additions & 8 deletions src/device-registry/bin/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
const isEmpty = require("is-empty");
const logger = log4js.getLogger(`${constants.ENVIRONMENT} -- bin/server`);
const { logText, logObject } = require("@utils/log");
const jsonify = require("@utils/jsonify");
const stringify = require("@utils/stringify");
require("@bin/jobs/store-signals-job");
require("@bin/jobs/new-store-readings-job");
// require("@bin/jobs/new-check-network-status-job");
Expand Down Expand Up @@ -90,28 +90,28 @@
errors: { message: err.message },
});
} else if (err.status === 400) {
logger.error(`Bad request error --- ${jsonify(err)}`);
logger.error(`Bad request error --- ${stringify(err)}`);

Check warning on line 93 in src/device-registry/bin/server.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/server.js#L93 

Added line #L93 was not covered by tests
res.status(err.status).json({
success: false,
message: "Bad request error",
errors: { message: err.message },
});
} else if (err.status === 401) {
logger.error(`Unauthorized --- ${jsonify(err)}`);
logger.error(`Unauthorized --- ${stringify(err)}`);

Check warning on line 100 in src/device-registry/bin/server.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/server.js#L100 

Added line #L100 was not covered by tests
res.status(err.status).json({
success: false,
message: "Unauthorized",
errors: { message: err.message },
});
} else if (err.status === 403) {
logger.error(`Forbidden --- ${jsonify(err)}`);
logger.error(`Forbidden --- ${stringify(err)}`);

Check warning on line 107 in src/device-registry/bin/server.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/server.js#L107 

Added line #L107 was not covered by tests
res.status(err.status).json({
success: false,
message: "Forbidden",
errors: { message: err.message },
});
} else if (err.status === 500) {
// logger.error(`Internal Server Error --- ${jsonify(err)}`);
// logger.error(`Internal Server Error --- ${stringify(err)}`);
// logger.error(`Stack Trace: ${err.stack}`);
logObject("the error", err);
res.status(err.status).json({
Expand All @@ -120,14 +120,14 @@
errors: { message: err.message },
});
} else if (err.status === 502 || err.status === 503 || err.status === 504) {
logger.error(`${err.message} --- ${jsonify(err)}`);
logger.error(`${err.message} --- ${stringify(err)}`);

Check warning on line 123 in src/device-registry/bin/server.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/server.js#L123 

Added line #L123 was not covered by tests
res.status(err.status).json({
success: false,
message: err.message,
errors: { message: err.message },
});
} else {
logger.error(`Internal Server Error --- ${jsonify(err)}`);
logger.error(`Internal Server Error --- ${stringify(err)}`);

Check warning on line 130 in src/device-registry/bin/server.js

View check run for this annotation

Codecov / codecov/patch

src/device-registry/bin/server.js#L130 

Added line #L130 was not covered by tests
logObject("Internal Server Error", err);
logger.error(`Stack Trace: ${err.stack}`);
res.status(err.status || 500).json({
Expand All @@ -138,7 +138,7 @@
}
} else {
logger.info(
`🍻🍻 HTTP response already sent to the client -- ${jsonify(err)}`
`🍻🍻 HTTP response already sent to the client -- ${stringify(err)}`
);
}
});
Expand Down
Loading
Loading