Feat: Improve Prometheus Metrics

docs/src/pages/guides/cli-options.md

@@ -245,13 +245,17 @@ Then you can scrape the metrics from `http://localhost:3001/metrics`.
 
 Monika exposes [Prometheus default metrics](https://prometheus.io/docs/instrumenting/writing_clientlibs/#standard-and-runtime-collectors), [Node.js specific metrics](https://github.com/siimon/prom-client/tree/master/lib/metrics), and Monika probe metrics below.
 
-| Metric Name                            | Type      | Purpose                                      | Label                                       |
-| -------------------------------------- | --------- | -------------------------------------------- | ------------------------------------------- |
-| `monika_probes_total`                  | Gauge     | Collect total probe                          | -                                           |
-| `monika_request_status_code_info`      | Gauge     | Collect HTTP status code                     | `id`, `name`, `url`, `method`               |
-| `monika_request_response_time_seconds` | Histogram | Collect duration of probe request in seconds | `id`, `name`, `url`, `method`, `statusCode` |
-| `monika_request_response_size_bytes`   | Gauge     | Collect size of response size in bytes       | `id`, `name`, `url`, `method`, `statusCode` |
-| `monika_alert_total`                   | Counter   | Collect total alert triggered                | `id`, `name`, `url`, `method`, `alertQuery` |
+| Metric Name                            | Type      | Purpose                                                   | Labels                                                |
+| -------------------------------------- | --------- | --------------------------------------------------------- | ----------------------------------------------------- |
+| `monika_alerts_triggered`              | Counter   | Collect count of alerts triggered by a probe              | `id`, `name`, `url`, `method`, `alertQuery`           |
+| `monika_alerts_triggered_total`        | Counter   | Collect total count of alerts triggered                   | -                                                     |
+| `monika_probes_running`                | Gauge     | Indicates whether a probe is running (1) or idle (0)      | `id`                                                  |
+| `monika_probes_running_total`          | Gauge     | Collect total count of probes currently running checks    | -                                                     |
+| `monika_probes_status`                 | Gauge     | Indicates the current status of a probe: 0 = DOWN, 1 = UP | `id`, `name`, `url`, `method`                         |
+| `monika_probes_total`                  | Gauge     | Collect total number of probes configured                 | -                                                     |
+| `monika_request_response_size_bytes`   | Gauge     | Collect size of the response in bytes                     | `id`, `name`, `url`, `method`, `statusCode`, `result` |
+| `monika_request_response_time_seconds` | Histogram | Collect duration of probe request in seconds              | `id`, `name`, `url`, `method`, `statusCode`, `result` |
+| `monika_request_status_code_info`      | Gauge     | Collect HTTP status code of the probe request             | `id`, `name`, `url`, `method`                         |
 
 ## Repeat
 

src/components/config/get.ts

@@ -22,6 +22,7 @@
  * SOFTWARE.                                                                      *
  **********************************************************************************/
 
+import { randomUUID } from 'node:crypto'
 import { getContext } from '../../context'
 import type { Config } from '../../interfaces/config'
 import { log } from '../../utils/pino'
@@ -41,7 +42,10 @@ export async function getRawConfig(): Promise<Config> {
     return addDefaultNotifications(config)
   }
 
-  return config
+  // Add default alerts for Probe not Accessible
+  const finalizedConfig = addDefaultAlerts(config)
+
+  return finalizedConfig
 }
 
 // mergeConfigs merges configs by overwriting each other
@@ -82,6 +86,27 @@ async function parseNativeConfig(): Promise<Config[]> {
   )
 }
 
+export const FAILED_REQUEST_ASSERTION = {
+  assertion: '',
+  message: 'Probe not accessible',
+}
+
+function addDefaultAlerts(config: Config) {
+  return {
+    ...config,
+    probes: config.probes.map((probe) => ({
+      ...probe,
+      alerts: [
+        ...(probe.alerts || []),
+        {
+          id: randomUUID(),
+          ...FAILED_REQUEST_ASSERTION,
+        },
+      ],
+    })),
+  }
+}
+
 async function parseNonNativeConfig(): Promise<Config | undefined> {
   const { flags } = getContext()
   const hasNonNativeConfig =

src/components/probe/prober/http/index.ts

@@ -159,6 +159,12 @@ export class HTTPProber extends BaseProber {
         response,
       })
 
+      getEventEmitter().emit(events.probe.status.changed, {
+        probe: this.probeConfig,
+        requestIndex,
+        status: 'up',
+      })
+
       this.logMessage(
         true,
         getProbeResultMessage({
@@ -226,6 +232,12 @@ export class HTTPProber extends BaseProber {
     }
     const alertId = getAlertID(url, validation, probeID)
 
+    getEventEmitter().emit(events.probe.status.changed, {
+      probe: this.probeConfig,
+      requestIndex,
+      status: 'down',
+    })
+
     getEventEmitter().emit(events.probe.alert.triggered, {
       probe: this.probeConfig,
       requestIndex,

src/components/probe/prober/index.ts

@@ -134,6 +134,7 @@ export abstract class BaseProber implements Prober {
 
       // this probe is definitely in incident state because of fail assertion, so send notification, etc.
       this.handleFailedProbe(probeResults)
+
       return
     }
 
@@ -148,6 +149,11 @@ export abstract class BaseProber implements Prober {
         requestIndex: index,
         response: requestResponse,
       })
+      getEventEmitter().emit(events.probe.status.changed, {
+        probe: this.probeConfig,
+        requestIndex: index,
+        status: 'up',
+      })
       logResponseTime(requestResponse.responseTime)
 
       if (

src/events/index.ts

@@ -34,6 +34,9 @@ export default {
     sanitized: 'CONFIG_SANITIZED',
     updated: 'CONFIG_UPDATED',
   },
+  notifications: {
+    sent: 'NOTIFICATIONS_SENT',
+  },
   probe: {
     alert: {
       triggered: 'PROBE_ALERT_TRIGGERED',
@@ -46,5 +49,8 @@ export default {
     notification: {
       willSend: 'PROBE_NOTIFICATION_WILL_SEND',
     },
+    status: {
+      changed: 'PROBE_STATUS_CHANGED',
+    },
   },
 }

src/loaders/index.ts

@@ -82,6 +82,7 @@ function initPrometheus(prometheusPort: number) {
     decrementProbeRunningTotal,
     incrementProbeRunningTotal,
     resetProbeRunningTotal,
+    collectProbeStatus,
   } = new PrometheusCollector()
 
   // collect prometheus metrics
@@ -93,6 +94,7 @@ function initPrometheus(prometheusPort: number) {
   eventEmitter.on(events.probe.ran, incrementProbeRunningTotal)
   eventEmitter.on(events.probe.finished, decrementProbeRunningTotal)
   eventEmitter.on(events.config.updated, resetProbeRunningTotal)
+  eventEmitter.on(events.probe.status.changed, collectProbeStatus)
 
   startPrometheusMetricsServer(prometheusPort)
 }