Some extra metrics for grafana

Dockerfile

@@ -7,7 +7,7 @@ ENV PYTHONUNBUFFERED 1
 
 COPY requirements.txt /usr/src/app
 
-RUN python -m pip install --upgrade pip
+# RUN python -m pip install --upgrade pip
 RUN pip install -r requirements.txt
 
 COPY . /usr/src/app

docker-compose.yml

@@ -1,9 +1,36 @@
 version: '3'
 
 services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    volumes:
+      - ./provisioning/prometheus:/etc/prometheus
+    ports:
+      - '9090:9090'
+  alertmanager:
+    image: prom/alertmanager
+    container_name: alertmanager
+    ports:
+      - '9093:9093'
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./provisioning/grafana/datasources:/etc/grafana/provisioning/datasources
+      - ./provisioning/grafana/dashboards:/etc/grafana/provisioning/dashboards
+  node-exporter: # addidional service as a exporter example
+    image: prom/node-exporter
+    container_name: node-exporter
+    ports:
+      - '9100:9100'
   web:
     restart: always
     build: .
+    environment:
+      ENABLE_PROMETHEUS: 1
     command: ./entrypoint.sh
     volumes:
       - .:/app

jobs/middlewares.py

@@ -1,19 +1,59 @@
+import time
+
 from django.utils.deprecation import MiddlewareMixin
 from django.utils.timezone import now
 
-from jobsapp.metrics import requests_total, last_user_activity_time
+from jobsapp.metrics import error_rates_counter, requests_total, last_user_activity_time, \
+    response_time_histogram
 
 
 class CustomMiddleware(MiddlewareMixin):
     def __init__(self, get_response):
         self.get_response = get_response
+        super().__init__(get_response)
 
-    def process_request(self, request):
-        path = request.path
-        user = request.user
-        if "metrics" not in path:
-            # because metrics is for prometheus url
+    def __call__(self, request):
+        # Exclude metrics endpoint from processing
+        if "metrics" not in request.path:
+            # Increment the Prometheus counter
             requests_total.labels(
-                endpoint=request.get_full_path, method=request.method, user=user
+                endpoint=request.get_full_path(),  # call the method to get the path
+                method=request.method,
+                user=request.user.get_username() if request.user.is_authenticated else 'Anonymous'
+                # use get_username() for the user label
             ).inc()
-        last_user_activity_time.labels(user=user).set(now().timestamp())
+
+            # Update the last user activity time
+            if request.user.is_authenticated:
+                last_user_activity_time.labels(user=request.user.get_username()).set(
+                    now().timestamp())
+
+        # Call the next middleware or view
+        response = self.get_response(request)
+        return response
+
+
+class ResponseTimeMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        start_time = time.time()
+        response = self.get_response(request)
+        response_time = time.time() - start_time
+
+        response_time_histogram.labels(method=request.method, endpoint=request.path).observe(
+            response_time)
+
+        return response
+
+
+class ErrorTrackingMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if 400 <= response.status_code < 600:
+            error_rates_counter.labels(status_code=response.status_code, endpoint=request.path).inc()
+        return response

jobs/settings.py

@@ -311,10 +311,12 @@
     "JWT_EXPIRATION_DELTA": timedelta(minutes=60),
 }
 
-ENABLE_PROMETHEUS = int(os.environ.get("ENABLE_PROMETHEUS", "0"))
+ENABLE_PROMETHEUS = 1
 
 if ENABLE_PROMETHEUS:
     INSTALLED_APPS += ["django_prometheus"]
     MIDDLEWARE = ['django_prometheus.middleware.PrometheusBeforeMiddleware'] + MIDDLEWARE + \
-                 ['django_prometheus.middleware.PrometheusAfterMiddleware']
+                 ['django_prometheus.middleware.PrometheusAfterMiddleware', "jobs.middlewares.CustomMiddleware"]
     MIDDLEWARE.append("jobs.middlewares.CustomMiddleware")
+    MIDDLEWARE.append("jobs.middlewares.ResponseTimeMiddleware")
+    MIDDLEWARE.append("jobs.middlewares.ErrorTrackingMiddleware")

jobsapp/metrics.py

@@ -1,4 +1,4 @@
-from prometheus_client import Counter, Enum, Gauge, Info, Summary
+from prometheus_client import Counter, Enum, Gauge, Histogram, Info, Summary
 
 info = Info(name="app", documentation="Information about the application")
 info.info({"version": "1.0", "language": "python", "framework": "django"})
@@ -13,3 +13,15 @@
     documentation="The last time when user was active.",
     labelnames=["user"],
 )
+
+response_time_histogram = Histogram(
+    name='app_response_time_seconds',
+    documentation='Response time for requests',
+    labelnames=['method', 'endpoint']
+)
+
+error_rates_counter = Counter(
+    name='app_error_rates_total',
+    documentation='The total number of errors',
+    labelnames=['status_code', 'endpoint']
+)