Publish latest code

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+snapper.iml
+snapper/
+config.json
+__pycache__
+/*.sh

README.md

@@ -0,0 +1,40 @@
+# :turtle: Snapper
+
+Snapper is a simple python script that executes [SnapRAID](https://github.com/amadvance/snapraid) in order to sync and scrub the array. Inspired by the great [snapraid-aio-script](https://github.com/auanasgheps/snapraid-aio-script) with a limited feature set.
+
+The reason I created this is that I wanted more granular control of how my setup worked, which consequently means, this script is opinionated.
+
+## Features
+
+- Sanity checks the array
+- Runs `touch` if necessary
+- Runs `diff` before attempting to `sync`
+- Allows you to pre-hash before syncing
+- Allows you to automatically re-run `sync` if snapraid recommends it
+- Allows you to run snapraid with a lower priority to keep server and drives responsive
+- Allows you to abort execution if configurable thresholds are broken
+- Allows you to `scrub` after `sync`
+- Logs the raw snapraid output as well as formatted text
+- Creates a nicely formatted report and sends it via email or discord
+- Provides live insight into the sync/scrub process in Discord
+- Spin down selected hard drives after script completion
+
+**This project is a work in progress, and can change at any time.**
+
+I welcome bugfixes and contributions, but be aware that I will not merge PRs that I do not feel do not fit the usage of this tool.
+
+## How to use
+
+- Ensure you have Python 3.7 or later installed
+- Install the necessary dependencies by running `pip3 install -r requirements.txt`
+- Download the [latest release](https://github.com/firasdib/snapper/releases/latest) of this project, or clone the git project.
+- Copy or rename `config.json.example` to `config.json`
+- Run the script via `python3 snapper.py`
+
+You may run the script with the `--force` flag to force a sync/scrub and ignore any thresholds or sanity checks.
+
+## Configuration
+
+A `config.json` file is required and expected to be in the same root as this script. 
+
+Please read through the [json schema](config.schema.json) to understand the exact details of each property. If you're not fluent in json schema (I don't blame you), you could use something like [this](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2Ffirasdib%2Fsnapper%2Fmain%2Fconfig.schema.json) to get a better idea of the different options.

config.json.example

@@ -0,0 +1,48 @@
+{
+  "snapraid": {
+    "binary": "/usr/bin/snapraid",
+    "config": "/etc/snapraid.conf",
+    "nice": 10,
+    "diff": {
+      "thresholds": {
+        "added": 500,
+        "removed": 500
+      }
+    },
+    "sync": {
+      "pre_hash": true,
+      "auto_sync": {
+        "enabled": false,
+        "max_attempts": 3
+      }
+    },
+    "scrub": {
+      "enabled": true,
+      "check_percent": 3,
+      "min_age": 30,
+      "scrub_new": true
+    }
+  },
+  "notifications": {
+    "email": {
+      "enabled": true,
+      "binary": "/usr/bin/mailx",
+      "from_email": "[email protected]",
+      "to_email": "[email protected]"
+    },
+    "discord": {
+      "enabled": false,
+      "webhook_id": "",
+      "webhook_token": ""
+    }
+  },
+  "logs": {
+    "dir": "/var/log/snapper",
+    "max_count": 14
+  },
+  "spindown": {
+    "enabled": true,
+    "binary": "/usr/sbin/hdparm",
+    "drives": "parity"
+  }
+}

config.schema.json

@@ -0,0 +1,252 @@
+{
+  "type": "object",
+  "properties": {
+    "snapraid": {
+      "type": "object",
+      "properties": {
+        "binary": {
+          "type": "string",
+          "examples": [
+            "/usr/bin/snapraid"
+          ],
+          "description": "The location of your snapraid executable."
+        },
+        "config": {
+          "type": "string",
+          "examples": [
+            "/etc/snapraid.conf"
+          ],
+          "description": "Location of the snapraid config file. Necessary for sanity checks."
+        },
+        "nice": {
+          "type": "number",
+          "examples": [
+            10
+          ],
+          "description": "Run snapraid at a given `nice`. By default processes run at `0`. Lower values mean higher priority. Ranges between -20 to +19.",
+          "minimum": -20,
+          "maximum": 19
+        },
+        "diff": {
+          "type": "object",
+          "properties": {
+            "thresholds": {
+              "type": "object",
+              "properties": {
+                "added": {
+                  "type": "number",
+                  "examples": [
+                    500
+                  ],
+                  "description": "If more files than the threshold amount have been added, don't execute jobs. Set to `0` to disable.",
+                  "minimum": 0
+                },
+                "removed": {
+                  "type": "number",
+                  "examples": [
+                    500
+                  ],
+                  "description": "If more files than the threshold amount have been removed, don't execute jobs. Set to `0` to disable.",
+                  "minimum": 0
+                }
+              },
+              "additionalProperties": false,
+              "required": ["added", "removed"]
+            }
+          },
+          "additionalProperties": false,
+          "required": ["thresholds"]
+        },
+        "sync": {
+          "type": "object",
+          "properties": {
+            "pre_hash": {
+              "type": "boolean",
+              "examples": [
+                true
+              ],
+              "description": "Whether to pre-hash changed blocks before syncing."
+            },
+            "auto_sync": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "examples": [
+                    false
+                  ],
+                  "description": "Whether or not to re-run the sync command if snapraid recommends it."
+                },
+                "max_attempts": {
+                  "type": "number",
+                  "examples": [3],
+                  "description": "The max amount of attempts to `sync` the array before bailing.",
+                  "minimum": 0
+                }
+              },
+              "additionalProperties": false,
+              "required": ["enabled", "max_attempts"]
+            }
+          },
+          "additionalProperties": false,
+          "required": ["pre_hash", "auto_sync"]
+        },
+        "scrub": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "examples": [
+                true
+              ],
+              "description": "Whether or not to scrub the array."
+            },
+            "check_percent": {
+              "type": "number",
+              "examples": [
+                3
+              ],
+              "description": "How many percent of the array to scrub each time. Set to `0` to disable scrubbing.",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "min_age": {
+              "type": "number",
+              "examples": [
+                30
+              ],
+              "description": "How old the blocks have to be before considered for scrub, in days.",
+              "minimum": 1
+            },
+            "scrub_new": {
+              "type": "boolean",
+              "examples": [
+                true
+              ],
+              "description": "Whether to scrub newly synced blocks or not."
+            }
+          },
+          "additionalProperties": false,
+          "required": ["enabled", "check_percent", "min_age", "scrub_new"]
+        }
+      },
+      "additionalProperties": false,
+      "required": ["binary", "config", "nice", "diff", "sync", "scrub"]
+    },
+    "notifications": {
+      "type": "object",
+      "properties": {
+        "email": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "examples": [
+                true
+              ],
+              "description": "Whether or not to send notifications and reports to the defined email."
+            },
+            "binary": {
+              "type": "string",
+              "examples": [
+                "/usr/bin/mailx"
+              ],
+              "description": "The location of `mailx`."
+            },
+            "from_email": {
+              "type": "string",
+              "examples": [
+                "[email protected]"
+              ],
+              "description": "The senders email."
+            },
+            "to_email": {
+              "type": "string",
+              "examples": [
+                "[email protected]"
+              ],
+              "description": "The recipients email."
+            }
+          },
+          "additionalProperties": false,
+          "required": ["enabled", "binary", "from_email", "to_email"]
+        },
+        "discord": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "examples": [
+                true
+              ],
+              "description": "Whether or not to send notifications and reports to Discord."
+            },
+            "webhook_id": {
+              "type": "string",
+              "examples": ["1234567890"],
+              "description": "Discord webhook id."
+            },
+            "webhook_token": {
+              "type": "string",
+              "examples": ["abc123"],
+              "description": "Discord webhook token."
+            }
+          },
+          "additionalProperties": false,
+          "required": ["enabled", "webhook_id", "webhook_token"]
+        }
+      },
+      "additionalProperties": false,
+      "required": ["email", "discord"]
+    },
+    "logs": {
+      "type": "object",
+      "properties": {
+        "dir": {
+          "type": "string",
+          "examples": [
+            "/var/log/snapper"
+          ],
+          "description": "The directory in which to save logs. Will be created if it does not exist."
+        },
+        "max_count": {
+          "type": "number",
+          "examples": [
+            14
+          ],
+          "description": "How many historic logs to keep. A new log is generated on each run, and the previous ones are rotated and gzipped.",
+          "minimum": 1
+        }
+      },
+      "additionalProperties": false,
+      "required": ["dir", "max_count"]
+    },
+    "spindown": {
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "examples": [true],
+          "description": "Whether to spin down hard drives after script execution or not"
+        },
+        "binary": {
+          "type": "string",
+          "examples": [
+            "/usr/sbin/hdparm"
+          ],
+          "description": "The location of the `hdparm` executable."
+        },
+        "drives": {
+          "type": "string",
+          "enum": ["parity", "all"],
+          "examples": ["parity"],
+          "description": "Which drives to spin down after script execution is complete"
+        }
+      },
+      "additionalProperties": false,
+      "required": ["enabled", "binary", "drives"]
+    }
+  },
+  "additionalProperties": false,
+  "required": ["snapraid", "notifications", "logs", "spindown"]
+}