feat: ovh3 pull off1 data + better sanoid_check

fixes: #306

confs/ovh3/sanoid/sanoid.conf

@@ -1,77 +1,42 @@
 # sanoid conf data - see https://github.com/jimsalterjrs/sanoid/wiki/Sanoid
+#
+# Excluded datasets, for now !
+# This will avoid sanoid_checks from checking them
+# no_sanoid_checks:rpool/storage:
+# no_sanoid_checks:rpool/subvol-100-disk-0:rpool/subvol-101-disk-0:rpool/subvol-102-disk-0:rpool/subvol-103-disk-0:
+# no_sanoid_checks:rpool/subvol-104-disk-0:rpool/subvol-105-disk-0:rpool/subvol-106-disk-0:rpool/subvol-107-disk-0:
+# no_sanoid_checks:rpool/subvol-108-disk-0:rpool/subvol-109-disk-0:rpool/subvol-110-disk-0:rpool/subvol-111-disk-0:
+# no_sanoid_checks:rpool/subvol-112-disk-0:rpool/subvol-113-disk-0:rpool/subvol-115-disk-0:rpool/subvol-116-disk-0:
+# no_sanoid_checks:rpool/subvol-117-disk-0:rpool/subvol-119-disk-0:rpool/subvol-120-disk-0:rpool/subvol-122-disk-0:
+# no_sanoid_checks:rpool/subvol-130-disk-0:rpool/subvol-140-disk-0:rpool/subvol-150-disk-0:rpool/subvol-150-disk-1:
+# no_sanoid_checks:rpool/vm-200-disk-0:rpool/vm-200-disk-0-old:rpool/vm-200-state-bef_alex_resize:rpool/vm-201-disk-0:
+# no_sanoid_checks:rpool/vm-202-disk-0:rpool/vm-202-state-bef_upgrade:rpool/vm-203-disk-0:rpool/vm-203-state-before_more_space:
+# no_sanoid_checks:rpool/backups:rpool/backups/monitoring-volumes:rpool/backups/off1-raphael
+# no_sanoid_checks:rpool/logs-nginx:
+# no_sanoid_checks:rpool/obf-old:rpool/opf-old: 
+# no_sanoid_checks:rpool/off/clonesrpool/off/clones/images:rpool/off/clones/orgs:rpool/off/clones/products:rpool/off/clones/users:
+# no_sanoid_checks:rpool/static:rpool/static/obf-images:rpool/static/off-pro-images:rpool/static/opf-images:rpool/static/opff-images:
+
 
 # obf
 [rpool/obf]
   use_template=synced_data
-  recursive=no
-
-[rpool/obf/cache]
-  use_template=synced_data
-  recursive=no
-
-[rpool/obf/html_data]
-  use_template=synced_data
-  recursive=no
-
-[rpool/obf/images]
-  use_template=synced_data
-  recursive=no
-
-[rpool/obf/products]
-  use_template=synced_data
-  recursive=no
+  recursive=yes
 
 # opf
 [rpool/opf]
   use_template=synced_data
-  recursive=no
-
-[rpool/opf/cache]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opf/html_data]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opf/images]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opf/products]
-  use_template=synced_data
-  recursive=no
-
+  recursive=yes
 
 # opff
 [rpool/opff]
   use_template=synced_data
-  recursive=no
-
-[rpool/opff/cache]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opff/html_data]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opff/images]
-  use_template=synced_data
-  recursive=no
-
-[rpool/opff/products]
-  use_template=synced_data
-  recursive=no
+  recursive=yes
 
 # off
-[rpool/off/users]
-  use_template=prod_data
-  recursive=no
-
-[rpool/off/images]
+[rpool/off]
   use_template=synced_data
-  recursive=no
+  recursive=yes
 
 # backups
 [rpool/backups/robotoff]
@@ -80,7 +45,20 @@
 
 # off2 pve volumes backup
 [rpool/backups/off2-pve]
-  use_template=synced_data
+  use_template=synced_sys
+  recursive=yes
+
+# off1 synced data
+[rpool/backups/off1-pve]
+  use_template=synced_sys
+  recursive=yes
+
+[rpool/backups/off1-pve-nvme]
+  use_template=synced_sys
+  recursive=yes
+
+[rpool/backups/off1-rpool]
+  use_template=synced_sys
   recursive=yes
 
 
@@ -104,6 +82,27 @@
   # Should old snapshots be pruned
   autoprune = yes
 
+# Template for system volumes
+[template_local_sys]
+  # How often snapshots should be taken under an hour
+  frequent_period=60
+  # What to keep
+  # number of sub-hourly snapshots
+  frequently = 0
+  # number of hourly snapshots
+  hourly = 10
+  # number of daily snashots
+  daily = 10
+  # number of monthly snashots
+  monthly = 6
+  # number of yearly snashots
+  yearly = 0
+  # whether snapshots should be taken automatically
+  autosnap = yes
+  # Should old snapshots be pruned
+  autoprune = yes
+
+
 # This template is for data that is synced from another machine
 # We do not create snapshots for they are provided by the source
 # We only prune snapshots to use a decent space
@@ -125,3 +124,24 @@
   # Should old snapshots be pruned
   autoprune = yes
 
+# policy for system data from remote servers
+# We only prune snapshots to use a decent space
+[template_synced_sys]
+  # How often snapshots should be taken under an hour
+  frequent_period=60
+  # What to keep
+  # number of sub-hourly snapshots
+  frequently = 0
+  # number of hourly snapshots
+  hourly = 10
+  # number of daily snashots
+  daily = 10
+  # number of monthly snashots
+  monthly = 6
+  # number of yearly snashots
+  yearly = 0
+  # whether snapshots should be taken automatically
+  autosnap = no
+  # Should old snapshots be pruned
+  autoprune = yes
+

confs/ovh3/sanoid/syncoid-args.conf

@@ -0,0 +1,28 @@
+# pulling from off2
+#
+# obf opf opff
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/obf rpool/obf 
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/opf rpool/opf
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/opff rpool/opff 
+# off and off-pro
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/off rpool/off 
+#--no-sync-snap --no-privilege-elevation [email protected]:zfs-nvme/off/products rpool/off/products
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/off-pro rpool/off-pro
+#--no-sync-snap --no-privilege-elevation [email protected]:zfs-nvme/off-pro/products rpool/off-pro/products
+#
+# PVE managed volumes
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/pve [email protected]:rpool/backups/off2-pve
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-nvme/pve [email protected]:rpool/backups/off2-pve-nvme
+# off2 root volume
+#--no-sync-snap --no-privilege-elevation --recursive [email protected]:rpool rpool/backups/off2-rpool
+
+## pulling from off1
+
+# PVE managed volumes, this includes mongodb data (in off1-pve-nvme)
+--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/pve rpool/backups/off1-pve
+--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-nvme/pve rpool/backups/off1-pve-nvme
+# off2 root volume
+--no-sync-snap --no-privilege-elevation --recursive [email protected]:rpool rpool/backups/off1-rpool
+
+
+

confs/ovh3/systemd/system/syncoid.service

@@ -0,0 +1 @@
+../../../common/systemd/system/syncoid.service

scripts/zfs/sanoid_check.sh

@@ -11,15 +11,17 @@ SNAP_MIN_DATE="7 hours ago"
 SANOID_MIN_DATE="7 hours ago"
 ERRORS=()
 
-function get_zfs_volumes {
- # get all sections but templates and remove brackets
- ZFS_VOLUMES=( \
-   $( \
-     cat /etc/sanoid/sanoid.conf | \
-     grep  '^\[.\+\]' | grep -v '\[template_' | \
-     sed -e 's/\[\(.\+\)\]/\1/' \
-   ) \
- );
+function get_zfs_datasets {
+ # list all zfs datasets
+ ZFS_POOLS=$(zpool list -o name -H)
+ ZFS_CANDIDATES_DATASETS=$(zfs list -o name -H -r $ZFS_POOLS)
+ # search for excluded datasests in sanoid.conf
+ # this is no_sanoid_checks:dataset/path/1:dataset/path/2
+ # we transform to :path1:path2:path3: for easy check if a value is inside
+ # it might be on several line
+ # Note: the line return in last tr is intentional to transform spaces and line return to ':'
+ EXCLUDED_DATASETS=":"$(grep -i  "no_sanoid_checks:" /etc/sanoid/sanoid.conf|tr -d " "|cut -d ":" --output-delimiter=" " -f 2-|tr " 
+" "::")":"
 }
 
 function check_last_snap_date {
@@ -52,10 +54,13 @@ function check_sanoid_run_date {
 # first check sanoid
 check_sanoid_run_date
 # then each volume
-get_zfs_volumes
-for volume in "${ZFS_VOLUMES[@]}"
+get_zfs_datasets
+for volume in $ZFS_DATASETS
 do
-  check_last_snap_date "$volume"
+  if [[ ! $EXCLUDED_DATASESTS =~ :$volume: ]]
+  then
+    check_last_snap_date "$volume"
+  fi
 done
 
 # if errors send email