diff --git a/grafana-backup/README.md b/grafana-backup/dashboard-exporter/README.md similarity index 94% rename from grafana-backup/README.md rename to grafana-backup/dashboard-exporter/README.md index cd8e5376..624615b7 100644 --- a/grafana-backup/README.md +++ b/grafana-backup/dashboard-exporter/README.md @@ -9,7 +9,7 @@ Then compresses and puts them to EOS folder `/eos/cms/store/group/offcomp_monit/ ``` { - "SECRET_KEY": "YOUR_SECRET_KEY" + "filesystem_exporter_token": "YOUR_SECRET_KEY" } ``` diff --git a/grafana-backup/dashboard-exporter.py b/grafana-backup/dashboard-exporter/dashboard-exporter.py similarity index 98% rename from grafana-backup/dashboard-exporter.py rename to grafana-backup/dashboard-exporter/dashboard-exporter.py index 938d22c8..68d14617 100755 --- a/grafana-backup/dashboard-exporter.py +++ b/grafana-backup/dashboard-exporter/dashboard-exporter.py @@ -25,7 +25,7 @@ def get_grafana_auth(fname): print(f"File {fname} does not exist") sys.exit(1) with open(fname, "r") as keyFile: - secret_key = json.load(keyFile).get("SECRET_KEY") + secret_key = json.load(keyFile).get("filesystem_exporter_token") headers = {"Authorization": f"Bearer {secret_key}"} return headers diff --git a/grafana-backup/run.sh b/grafana-backup/dashboard-exporter/run.sh similarity index 100% rename from grafana-backup/run.sh rename to grafana-backup/dashboard-exporter/run.sh diff --git a/grafana-backup/production-copy/README.md b/grafana-backup/production-copy/README.md new file mode 100644 index 00000000..f513e14c --- /dev/null +++ b/grafana-backup/production-copy/README.md @@ -0,0 +1,33 @@ + +## Production Copy +This script copies Grafana dashboards from the folder "Production" to "Production Copy" using the Grafana API. + +## Requirements +- Create a file for your Grafana authentication token and name it `keys.json` + +``` +{ + "production_copy_token": "YOUR_SECRET_KEY" +} +``` + +- Get `amtool` executable +``` +curl -ksLO https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz && \ +tar xfz alertmanager-0.27.0.linux-amd64.tar.gz && \ +mv alertmanager-0.27.0.linux-amd64/amtool . && \ +rm -rf alertmanager-0.27.0.linux-amd64* +``` + +## How to use + +- Run the file: +```sh +python3 dashboard-copy.py --url https://grafana-url.com --token keys.json +``` +- Set the file executable: +```sh +chmod +x ./dashboard-copy.py +``` +- Set crontab for preferred timeframe +- See `run.sh` file \ No newline at end of file diff --git a/grafana-backup/production-copy/dashboard-copy.py b/grafana-backup/production-copy/dashboard-copy.py new file mode 100644 index 00000000..a221d96b --- /dev/null +++ b/grafana-backup/production-copy/dashboard-copy.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +This script copies Grafana dashboards from one folder to another. +""" +import datetime +import json +import logging +import sys + +import click +import requests + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + +def get_grafana_auth(fname): + """Load Grafana authentication token from a file.""" + with open(fname, "r") as token_file: + token = json.load(token_file)["production_copy_token"] + return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + +def get_folder_id(base_url, headers, folder_name): + """Fetch the folder ID for a given folder name.""" + response = requests.get(f"{base_url}/api/folders", headers=headers) + response.raise_for_status() + for folder in response.json(): + if folder["title"] == folder_name: + return folder["id"] + return None + + +def get_folder_uid(base_url, headers, folder_name): + """Fetch the folder UID for a given folder name.""" + response = requests.get(f"{base_url}/api/folders", headers=headers) + response.raise_for_status() + for folder in response.json(): + if folder["title"] == folder_name: + return folder["uid"] + return None + + +def get_folder_permissions(base_url, headers, folder_uid): + """Fetch the permissions of a folder.""" + response = requests.get(f"{base_url}/api/folders/{folder_uid}/permissions", headers=headers) + response.raise_for_status() + return response.json() + + +def update_folder_permissions(base_url, headers, folder_uid, permissions): + """Set permissions for a folder.""" + payload = {"items": permissions} + response = requests.post(f"{base_url}/api/folders/{folder_uid}/permissions", headers=headers, json=payload) + response.raise_for_status() + logging.info(f"Permissions for folder UID '{folder_uid}' have been updated.") + + +def delete_folder(base_url, headers, folder_uid): + """Delete a folder by its UID.""" + response = requests.delete(f"{base_url}/api/folders/{folder_uid}", headers=headers) + response.raise_for_status() + logging.info(f"Deleted folder with UID {folder_uid}.") + + +def create_folder(base_url, headers, folder_name): + """Create a folder. If it exists, delete and recreate it while preserving permissions.""" + folder_uid = get_folder_uid(base_url, headers, folder_name) + preserved_permissions = [] + + if folder_uid: + logging.info(f"Folder '{folder_name}' exists. Fetching permissions and overwriting...") + preserved_permissions = get_folder_permissions(base_url, headers, folder_uid) + delete_folder(base_url, headers, folder_uid) + + response = requests.post(f"{base_url}/api/folders", headers=headers, json={"title": folder_name}) + response.raise_for_status() + folder = response.json() + logging.info(f"Created folder '{folder_name}'.") + + if preserved_permissions: + update_folder_permissions(base_url, headers, folder["uid"], preserved_permissions) + + return folder["id"] + + +def get_dashboards_in_folder(base_url, headers, folder_id): + """Fetch all dashboards in a folder.""" + response = requests.get(f"{base_url}/api/search?folderIds={folder_id}&type=dash-db", headers=headers) + response.raise_for_status() + return response.json() + + +def copy_dashboard(base_url, headers, dashboard, target_folder_id): + """Copy a dashboard to a new folder.""" + dashboard_uid = dashboard["uid"] + response = requests.get(f"{base_url}/api/dashboards/uid/{dashboard_uid}", headers=headers) + response.raise_for_status() + dashboard_data = response.json() + dashboard_data["dashboard"]["id"] = None + dashboard_data["dashboard"]["uid"] = None + dashboard_data["folderId"] = target_folder_id + dashboard_data["message"] = f"Copied on {datetime.datetime.now()}" + response = requests.post(f"{base_url}/api/dashboards/db", headers=headers, json=dashboard_data) + response.raise_for_status() + logging.info(f"Copied dashboard '{dashboard['title']}' to folder ID {target_folder_id}.") + + +@click.command() +@click.option("--url", required=True, help="Base URL of the Grafana instance") +@click.option("--token", "fname", required=True, help="API or Service Account token for authentication") +def main(url, fname): + headers = get_grafana_auth(fname) + source_folder_name = "Production" + target_folder_name = "Production Copy" + + source_folder_id = get_folder_id(url, headers, source_folder_name) + if not source_folder_id: + logging.error(f"Source folder '{source_folder_name}' does not exist.") + return + + target_folder_id = create_folder(url, headers, target_folder_name) + dashboards = get_dashboards_in_folder(url, headers, source_folder_id) + logging.info(f"Found {len(dashboards)} dashboards in '{source_folder_name}'.") + + for dashboard in dashboards: + copy_dashboard(url, headers, dashboard, target_folder_id) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + logging.error(f"An unexpected error occurred: {e}") + sys.exit(1) diff --git a/grafana-backup/production-copy/run.sh b/grafana-backup/production-copy/run.sh new file mode 100755 index 00000000..598893c8 --- /dev/null +++ b/grafana-backup/production-copy/run.sh @@ -0,0 +1,49 @@ +#!/bin/bash -l + +set -e + +# This script copies Grafana dashboards from the folder "Production" to "Production Copy". +##H Usage: run.sh GRAFANA_URL API_TOKEN +##H Example: +##H run.sh https://grafana-url.com your-api-token + +if [ "$1" == "-h" ] || [ "$1" == "-help" ] || [ "$1" == "--help" ] || [ "$1" == "help" ] || [ "$1" == "" ]; then + grep "^##H" <"$0" | sed -e "s,##H,,g" + exit 1 +fi + +# Arguments +GRAFANA_URL="$1" +API_TOKEN="$2" + +addr=cms-comp-monit-alerts@cern.ch + +trap onExit EXIT + +function onExit() { + local status=$? + if [ $status -ne 0 ]; then + local msg="Grafana dashboard copy cron failure. Please see Kubernetes 'cron' cluster logs." + if [ -x ./amtool ]; then + expire=$(date -d '+1 hour' --rfc-3339=ns | tr ' ' 'T') + local urls="http://cms-monitoring.cern.ch:30093 http://cms-monitoring-ha1.cern.ch:30093 http://cms-monitoring-ha2.cern.ch:30093" + for url in $urls; do + ./amtool alert add grafana_dashboard_copy_failure \ + alertname=grafana_dashboard_copy_failure severity=monitoring tag=cronjob alert=amtool \ + --end="$expire" \ + --annotation=summary="$msg" \ + --annotation=date="$(date)" \ + --annotation=hostname="$(hostname)" \ + --annotation=status="$status" \ + --alertmanager.url="$url" + done + else + echo "$msg" | mail -s "Cron alert grafana_dashboard_copy_failure" "$addr" + fi + fi +} + +cd "$(dirname "$0")" || exit + +# Execute the Python script +python3 dashboard-copy.py --url "$GRAFANA_URL" --token "$API_TOKEN" \ No newline at end of file