Merge pull request #282 from nikodemas/add_grafana_backup

Add grafana production copy cronjob

grafana-backup/README.md → grafana-backup/dashboard-exporter/README.md

@@ -9,7 +9,7 @@ Then compresses and puts them to EOS folder `/eos/cms/store/group/offcomp_monit/
 
 ```
     {
-        "SECRET_KEY": "YOUR_SECRET_KEY"
+        "filesystem_exporter_token": "YOUR_SECRET_KEY"
     }
 ```
 

grafana-backup/dashboard-exporter.py → grafana-backup/dashboard-exporter/dashboard-exporter.py

@@ -25,7 +25,7 @@ def get_grafana_auth(fname):
         print(f"File {fname} does not exist")
         sys.exit(1)
     with open(fname, "r") as keyFile:
-        secret_key = json.load(keyFile).get("SECRET_KEY")
+        secret_key = json.load(keyFile).get("filesystem_exporter_token")
     headers = {"Authorization": f"Bearer {secret_key}"}
     return headers
 

grafana-backup/production-copy/README.md

@@ -0,0 +1,33 @@
+
+## Production Copy
+This script copies Grafana dashboards from the folder "Production" to "Production Copy" using the Grafana API.
+
+## Requirements
+- Create a file for your Grafana authentication token and name it `keys.json`
+
+```
+{
+    "production_copy_token": "YOUR_SECRET_KEY"
+}
+```
+
+- Get `amtool` executable
+```
+curl -ksLO https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz && \
+tar xfz alertmanager-0.27.0.linux-amd64.tar.gz && \
+mv alertmanager-0.27.0.linux-amd64/amtool . && \
+rm -rf alertmanager-0.27.0.linux-amd64*
+```
+
+## How to use
+
+- Run the file:
+```sh
+python3 dashboard-copy.py --url https://grafana-url.com --token keys.json
+```
+- Set the file executable:
+```sh
+chmod +x ./dashboard-copy.py
+```
+- Set crontab for preferred timeframe
+- See `run.sh` file

grafana-backup/production-copy/dashboard-copy.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This script copies Grafana dashboards from one folder to another.
+"""
+import datetime
+import json
+import logging
+import sys
+
+import click
+import requests
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+
+def get_grafana_auth(fname):
+    """Load Grafana authentication token from a file."""
+    with open(fname, "r") as token_file:
+        token = json.load(token_file)["production_copy_token"]
+    return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+
+
+def get_folder_id(base_url, headers, folder_name):
+    """Fetch the folder ID for a given folder name."""
+    response = requests.get(f"{base_url}/api/folders", headers=headers)
+    response.raise_for_status()
+    for folder in response.json():
+        if folder["title"] == folder_name:
+            return folder["id"]
+    return None
+
+
+def get_folder_uid(base_url, headers, folder_name):
+    """Fetch the folder UID for a given folder name."""
+    response = requests.get(f"{base_url}/api/folders", headers=headers)
+    response.raise_for_status()
+    for folder in response.json():
+        if folder["title"] == folder_name:
+            return folder["uid"]
+    return None
+
+
+def get_folder_permissions(base_url, headers, folder_uid):
+    """Fetch the permissions of a folder."""
+    response = requests.get(f"{base_url}/api/folders/{folder_uid}/permissions", headers=headers)
+    response.raise_for_status()
+    return response.json()
+
+
+def update_folder_permissions(base_url, headers, folder_uid, permissions):
+    """Set permissions for a folder."""
+    payload = {"items": permissions}
+    response = requests.post(f"{base_url}/api/folders/{folder_uid}/permissions", headers=headers, json=payload)
+    response.raise_for_status()
+    logging.info(f"Permissions for folder UID '{folder_uid}' have been updated.")
+
+
+def delete_folder(base_url, headers, folder_uid):
+    """Delete a folder by its UID."""
+    response = requests.delete(f"{base_url}/api/folders/{folder_uid}", headers=headers)
+    response.raise_for_status()
+    logging.info(f"Deleted folder with UID {folder_uid}.")
+
+
+def create_folder(base_url, headers, folder_name):
+    """Create a folder. If it exists, delete and recreate it while preserving permissions."""
+    folder_uid = get_folder_uid(base_url, headers, folder_name)
+    preserved_permissions = []
+
+    if folder_uid:
+        logging.info(f"Folder '{folder_name}' exists. Fetching permissions and overwriting...")
+        preserved_permissions = get_folder_permissions(base_url, headers, folder_uid)
+        delete_folder(base_url, headers, folder_uid)
+
+    response = requests.post(f"{base_url}/api/folders", headers=headers, json={"title": folder_name})
+    response.raise_for_status()
+    folder = response.json()
+    logging.info(f"Created folder '{folder_name}'.")
+
+    if preserved_permissions:
+        update_folder_permissions(base_url, headers, folder["uid"], preserved_permissions)
+
+    return folder["id"]
+
+
+def get_dashboards_in_folder(base_url, headers, folder_id):
+    """Fetch all dashboards in a folder."""
+    response = requests.get(f"{base_url}/api/search?folderIds={folder_id}&type=dash-db", headers=headers)
+    response.raise_for_status()
+    return response.json()
+
+
+def copy_dashboard(base_url, headers, dashboard, target_folder_id):
+    """Copy a dashboard to a new folder."""
+    dashboard_uid = dashboard["uid"]
+    response = requests.get(f"{base_url}/api/dashboards/uid/{dashboard_uid}", headers=headers)
+    response.raise_for_status()
+    dashboard_data = response.json()
+    dashboard_data["dashboard"]["id"] = None
+    dashboard_data["dashboard"]["uid"] = None
+    dashboard_data["folderId"] = target_folder_id
+    dashboard_data["message"] = f"Copied on {datetime.datetime.now()}"
+    response = requests.post(f"{base_url}/api/dashboards/db", headers=headers, json=dashboard_data)
+    response.raise_for_status()
+    logging.info(f"Copied dashboard '{dashboard['title']}' to folder ID {target_folder_id}.")
+
+
+@click.command()
+@click.option("--url", required=True, help="Base URL of the Grafana instance")
+@click.option("--token", "fname", required=True, help="API or Service Account token for authentication")
+def main(url, fname):
+    headers = get_grafana_auth(fname)
+    source_folder_name = "Production"
+    target_folder_name = "Production Copy"
+
+    source_folder_id = get_folder_id(url, headers, source_folder_name)
+    if not source_folder_id:
+        logging.error(f"Source folder '{source_folder_name}' does not exist.")
+        return
+
+    target_folder_id = create_folder(url, headers, target_folder_name)
+    dashboards = get_dashboards_in_folder(url, headers, source_folder_id)
+    logging.info(f"Found {len(dashboards)} dashboards in '{source_folder_name}'.")
+
+    for dashboard in dashboards:
+        copy_dashboard(url, headers, dashboard, target_folder_id)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {e}")
+        sys.exit(1)

grafana-backup/production-copy/run.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -l
+
+set -e
+
+# This script copies Grafana dashboards from the folder "Production" to "Production Copy".
+##H Usage: run.sh GRAFANA_URL API_TOKEN
+##H Example:
+##H        run.sh https://grafana-url.com your-api-token
+
+if [ "$1" == "-h" ] || [ "$1" == "-help" ] || [ "$1" == "--help" ] || [ "$1" == "help" ] || [ "$1" == "" ]; then
+    grep "^##H" <"$0" | sed -e "s,##H,,g"
+    exit 1
+fi
+
+# Arguments
+GRAFANA_URL="$1"
+API_TOKEN="$2"
+
+[email protected]
+
+trap onExit EXIT
+
+function onExit() {
+  local status=$?
+  if [ $status -ne 0 ]; then
+    local msg="Grafana dashboard copy cron failure. Please see Kubernetes 'cron' cluster logs."
+    if [ -x ./amtool ]; then
+      expire=$(date -d '+1 hour' --rfc-3339=ns | tr ' ' 'T')
+      local urls="http://cms-monitoring.cern.ch:30093 http://cms-monitoring-ha1.cern.ch:30093 http://cms-monitoring-ha2.cern.ch:30093"
+      for url in $urls; do
+        ./amtool alert add grafana_dashboard_copy_failure \
+          alertname=grafana_dashboard_copy_failure severity=monitoring tag=cronjob alert=amtool \
+          --end="$expire" \
+          --annotation=summary="$msg" \
+          --annotation=date="$(date)" \
+          --annotation=hostname="$(hostname)" \
+          --annotation=status="$status" \
+          --alertmanager.url="$url"
+      done
+    else
+      echo "$msg" | mail -s "Cron alert grafana_dashboard_copy_failure" "$addr"
+    fi
+  fi
+}
+
+cd "$(dirname "$0")" || exit
+
+# Execute the Python script
+python3 dashboard-copy.py --url "$GRAFANA_URL" --token "$API_TOKEN"