From 2557dfc8acc4af4f2f0101a4e476ceb0aaefb04d Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Fri, 1 Dec 2023 23:10:46 +0200 Subject: [PATCH 1/9] Docker: run npm-ci with less context to cache it better This significantly improves repeat dev build speed and saves hitting external resources. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ae5cd4f..301d408 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM node:18 AS build-env -COPY . /app +COPY package.json package-lock.json /app/ WORKDIR /app RUN npm ci --no-save +COPY . /app RUN npm run build # re-install without dev dependencies RUN npm ci --omit=dev From cb14536731accd5fd84a01f0c6ea0d780cf141f2 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Fri, 1 Dec 2023 23:41:41 +0200 Subject: [PATCH 2/9] Exit the server on the interrupt signal This makes ^c work when for example running with `docker -it` --- packages/server/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 521a9bc..52af056 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -38,6 +38,10 @@ process log.error(`${inspect(err)} Uncaught Exception thrown`); process.exit(1); + }) + .on('SIGINT', function() { + console.log("Caught interrupt signal"); + process.exit(); }); wssMitm.on('connection', (ws, req) => { From 514bca2dd18ce61f8c40be9954cc5400ea7c15b9 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Mon, 4 Dec 2023 14:32:28 +0200 Subject: [PATCH 3/9] Change `rotom_devies` metric to report only connected devices --- packages/server/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 52af056..916dce7 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -323,7 +323,7 @@ if (config.logging.consoleStatus) { setInterval(() => { let connectedDevices = 0; // set memory - Object.entries(controlConnections).forEach(([, connection]) => { + Object.entries(controlConnections).filter(([, connection]) => connection.isAlive).forEach(([, connection]) => { const origin = connection?.origin || 'Unknown'; connectedDevices += 1; From bee37db7d3ee345c7e496ef5aeb3f5d393e2c687 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Mon, 4 Dec 2023 16:13:51 +0200 Subject: [PATCH 4/9] Only record alive workers in `rotom_workers` --- packages/server/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 916dce7..044e043 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -344,12 +344,13 @@ setInterval(() => { if (!(origin in originActiveWorkers)) { originActiveWorkers[origin] = 0; } - if (connection.scanner) { + if (connection.scanner && connection.scanner.isAlive && connection.mitm.isAlive) { originActiveWorkers[origin] += 1; } }); // set workers + workersGauge.reset(); // Otherwise dropped devices stay stale Object.entries(originActiveWorkers).forEach(([name, number]) => { const validNumber = Number.isFinite(number) ? number : 0; workersGauge.labels(name).set(validNumber); From 6dfeb3af425bba4974516a2b2d29aa0ec2b31c75 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Mon, 4 Dec 2023 16:33:05 +0200 Subject: [PATCH 5/9] Prettier my changes --- packages/server/src/index.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 044e043..09afc00 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -39,8 +39,8 @@ process process.exit(1); }) - .on('SIGINT', function() { - console.log("Caught interrupt signal"); + .on('SIGINT', function () { + console.log('Caught interrupt signal'); process.exit(); }); @@ -323,17 +323,19 @@ if (config.logging.consoleStatus) { setInterval(() => { let connectedDevices = 0; // set memory - Object.entries(controlConnections).filter(([, connection]) => connection.isAlive).forEach(([, connection]) => { - const origin = connection?.origin || 'Unknown'; - connectedDevices += 1; - - const validMemFree = Number.isFinite(connection.lastMemory.memFree) ? connection.lastMemory.memFree : 0; - deviceMemoryFree.labels(origin).set(validMemFree); - const validMemMitm = Number.isFinite(connection.lastMemory.memMitm) ? connection.lastMemory.memMitm : 0; - deviceMemoryMitm.labels(origin).set(validMemMitm); - const validMemStart = Number.isFinite(connection.lastMemory.memStart) ? connection.lastMemory.memStart : 0; - deviceMemoryStart.labels(origin).set(validMemStart); - }); + Object.entries(controlConnections) + .filter(([, connection]) => connection.isAlive) + .forEach(([, connection]) => { + const origin = connection?.origin || 'Unknown'; + connectedDevices += 1; + + const validMemFree = Number.isFinite(connection.lastMemory.memFree) ? connection.lastMemory.memFree : 0; + deviceMemoryFree.labels(origin).set(validMemFree); + const validMemMitm = Number.isFinite(connection.lastMemory.memMitm) ? connection.lastMemory.memMitm : 0; + deviceMemoryMitm.labels(origin).set(validMemMitm); + const validMemStart = Number.isFinite(connection.lastMemory.memStart) ? connection.lastMemory.memStart : 0; + deviceMemoryStart.labels(origin).set(validMemStart); + }); // set number of active devices (couldn't get it correct with add/removal signals ; missed some decrement) devicesGauge.set(connectedDevices); From ad4c64e299e59dd336cffd582b8b2f0adb7cd4d6 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Tue, 5 Dec 2023 14:34:48 +0200 Subject: [PATCH 6/9] Split device and worker metrics to _total & _alive variants --- packages/server/src/index.ts | 43 +++++++++++++++++++++++++++--------- packages/server/src/utils.ts | 25 ++++++++++++++++----- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 09afc00..dd9d9cc 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -7,8 +7,10 @@ import { WebSocketServer } from 'ws'; import { StatusDTO, WorkerDTO, JobsDTO, JobsStatusDTO } from '@rotom/types'; import { promRegistry, - workersGauge, - devicesGauge, + workersTotalGauge, + workersAliveGauge, + devicesTotalGauge, + devicesAliveGauge, deviceMemoryFree, deviceMemoryMitm, deviceMemoryStart, @@ -83,8 +85,8 @@ wssMitm.on('connection', (ws, req) => { restartRequired = true; } if (memoryStatus.memStart) { - const prefix = Object.keys(config.monitor.maxMemStartMultipleOverwrite).find((key) => - mitm.origin?.startsWith(key), + const prefix = Object.keys(config.monitor.maxMemStartMultipleOverwrite).find( + (key) => mitm.origin?.startsWith(key), ); const value = prefix @@ -320,9 +322,19 @@ if (config.logging.consoleStatus) { } // prometheus helper +// This is not the Best way to do this, but previous attemps at active tracking via events have failed setInterval(() => { let connectedDevices = 0; - // set memory + let totalDevices = 0; + + Object.entries(controlConnections).forEach(([, connection]) => { + const origin = connection?.origin || 'Unknown'; + totalDevices += 1; + }); + // set memory for alive devices, but first reset to get rid of dropped origins + deviceMemoryFree.reset(); + deviceMemoryMitm.reset(); + deviceMemoryStart.reset(); Object.entries(controlConnections) .filter(([, connection]) => connection.isAlive) .forEach(([, connection]) => { @@ -336,26 +348,37 @@ setInterval(() => { const validMemStart = Number.isFinite(connection.lastMemory.memStart) ? connection.lastMemory.memStart : 0; deviceMemoryStart.labels(origin).set(validMemStart); }); - // set number of active devices (couldn't get it correct with add/removal signals ; missed some decrement) - devicesGauge.set(connectedDevices); + devicesAliveGauge.set(connectedDevices); + devicesTotalGauge.set(totalDevices); // fetch active workers const originActiveWorkers: Record = {}; + const originTotalWorkers: Record = {}; Object.entries(currentConnections).forEach(([, connection]) => { const origin = connection.mitm?.origin || 'Unknown'; if (!(origin in originActiveWorkers)) { originActiveWorkers[origin] = 0; } + if (!(origin in originTotalWorkers)) { + originTotalWorkers[origin] = 0; + } + + originTotalWorkers[origin] += 1; if (connection.scanner && connection.scanner.isAlive && connection.mitm.isAlive) { originActiveWorkers[origin] += 1; } }); - // set workers - workersGauge.reset(); // Otherwise dropped devices stay stale + // set workers, but first reset to get rid of dropped origins + workersTotalGauge.reset(); + workersAliveGauge.reset(); + Object.entries(originTotalWorkers).forEach(([name, number]) => { + const validNumber = Number.isFinite(number) ? number : 0; + workersTotalGauge.labels(name).set(validNumber); + }); Object.entries(originActiveWorkers).forEach(([name, number]) => { const validNumber = Number.isFinite(number) ? number : 0; - workersGauge.labels(name).set(validNumber); + workersAliveGauge.labels(name).set(validNumber); }); }, 5000); diff --git a/packages/server/src/utils.ts b/packages/server/src/utils.ts index bf0a683..d1aa0ea 100644 --- a/packages/server/src/utils.ts +++ b/packages/server/src/utils.ts @@ -5,15 +5,28 @@ collectDefaultMetrics({ register: promRegistry }); const prefix = 'rotom_'; -export const devicesGauge = new Gauge({ - name: prefix + 'devices', - help: 'Devices in use', +export const devicesTotalGauge = new Gauge({ + name: prefix + 'devices_total', + help: 'Devices known regardless of state', registers: [promRegistry], }); -export const workersGauge = new Gauge({ - name: prefix + 'workers', - help: 'Workers in use', +export const devicesAliveGauge = new Gauge({ + name: prefix + 'devices_alive', + help: 'Devices that pass isAlive', + registers: [promRegistry], +}); + +export const workersTotalGauge = new Gauge({ + name: prefix + 'workers_total', + help: 'Workers known regardless of state', + labelNames: ['origin'], + registers: [promRegistry], +}); + +export const workersAliveGauge = new Gauge({ + name: prefix + 'workers_alive', + help: 'Workers that pass isAlive', labelNames: ['origin'], registers: [promRegistry], }); From 0e81e85820a07e6fa0c2cfed88f352b6d8600007 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Tue, 5 Dec 2023 14:40:49 +0200 Subject: [PATCH 7/9] Drop useless copypasted variable. --- packages/server/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index dd9d9cc..f1fb92a 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -328,7 +328,6 @@ setInterval(() => { let totalDevices = 0; Object.entries(controlConnections).forEach(([, connection]) => { - const origin = connection?.origin || 'Unknown'; totalDevices += 1; }); // set memory for alive devices, but first reset to get rid of dropped origins From 1584f40be6d8d0bab4fe898739712ed776352d32 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Tue, 5 Dec 2023 14:57:27 +0200 Subject: [PATCH 8/9] Simplify counting total devices, make linter happy --- packages/server/src/index.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index f1fb92a..fc851fe 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -85,8 +85,8 @@ wssMitm.on('connection', (ws, req) => { restartRequired = true; } if (memoryStatus.memStart) { - const prefix = Object.keys(config.monitor.maxMemStartMultipleOverwrite).find( - (key) => mitm.origin?.startsWith(key), + const prefix = Object.keys(config.monitor.maxMemStartMultipleOverwrite).find((key) => + mitm.origin?.startsWith(key), ); const value = prefix @@ -325,11 +325,7 @@ if (config.logging.consoleStatus) { // This is not the Best way to do this, but previous attemps at active tracking via events have failed setInterval(() => { let connectedDevices = 0; - let totalDevices = 0; - Object.entries(controlConnections).forEach(([, connection]) => { - totalDevices += 1; - }); // set memory for alive devices, but first reset to get rid of dropped origins deviceMemoryFree.reset(); deviceMemoryMitm.reset(); @@ -347,8 +343,10 @@ setInterval(() => { const validMemStart = Number.isFinite(connection.lastMemory.memStart) ? connection.lastMemory.memStart : 0; deviceMemoryStart.labels(origin).set(validMemStart); }); + + // Set device counts + devicesTotalGauge.set(Object.keys(controlConnections).length); devicesAliveGauge.set(connectedDevices); - devicesTotalGauge.set(totalDevices); // fetch active workers const originActiveWorkers: Record = {}; From f5b727b7ed1cd45fe3d664ff7d19e719c2396843 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Tue, 5 Dec 2023 17:41:32 +0200 Subject: [PATCH 9/9] Rename metric to `rotom_workers_active` for UI consistency --- packages/server/src/index.ts | 6 +++--- packages/server/src/utils.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index fc851fe..da0ae2b 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -8,7 +8,7 @@ import { StatusDTO, WorkerDTO, JobsDTO, JobsStatusDTO } from '@rotom/types'; import { promRegistry, workersTotalGauge, - workersAliveGauge, + workersActiveGauge, devicesTotalGauge, devicesAliveGauge, deviceMemoryFree, @@ -368,14 +368,14 @@ setInterval(() => { // set workers, but first reset to get rid of dropped origins workersTotalGauge.reset(); - workersAliveGauge.reset(); + workersActiveGauge.reset(); Object.entries(originTotalWorkers).forEach(([name, number]) => { const validNumber = Number.isFinite(number) ? number : 0; workersTotalGauge.labels(name).set(validNumber); }); Object.entries(originActiveWorkers).forEach(([name, number]) => { const validNumber = Number.isFinite(number) ? number : 0; - workersAliveGauge.labels(name).set(validNumber); + workersActiveGauge.labels(name).set(validNumber); }); }, 5000); diff --git a/packages/server/src/utils.ts b/packages/server/src/utils.ts index d1aa0ea..82fd7ad 100644 --- a/packages/server/src/utils.ts +++ b/packages/server/src/utils.ts @@ -24,9 +24,9 @@ export const workersTotalGauge = new Gauge({ registers: [promRegistry], }); -export const workersAliveGauge = new Gauge({ - name: prefix + 'workers_alive', - help: 'Workers that pass isAlive', +export const workersActiveGauge = new Gauge({ + name: prefix + 'workers_active', + help: 'Workers that have an active mitm connection', labelNames: ['origin'], registers: [promRegistry], });