From 6f913a5e5a979f5f51ba7eeee949263179942d38 Mon Sep 17 00:00:00 2001 From: Matheson Steplock Date: Thu, 23 May 2024 21:53:56 +0000 Subject: [PATCH 1/5] Update SFTP/SCP and rsync error handling in run.sh --- remote-backup/run.sh | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/remote-backup/run.sh b/remote-backup/run.sh index d4c5935..be105a0 100755 --- a/remote-backup/run.sh +++ b/remote-backup/run.sh @@ -202,17 +202,13 @@ function copy-backup-to-remote { fi bashio::log.info "Copying backup using SFTP/SCP." - ( sshpass -p "${REMOTE_PASSWORD}" \ - scp ${DEBUG_FLAG:-} -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || ( + scp ${DEBUG_FLAG:-} -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || bashio::log.warning "SFTP transfer failed, falling back to SCP: $(sshpass_error $?)" sshpass -p "${REMOTE_PASSWORD}" \ - scp ${DEBUG_FLAG:-} -O -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || ( - bashio::log.error "Error copying backup ${SLUG}.tar to ${remote_directory} on ${REMOTE_HOST}: $(sshpass_error $?)" + scp ${DEBUG_FLAG:-} -O -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || + bashio::log.error "Error copying backup ${SLUG}.tar to ${remote_directory} on ${REMOTE_HOST}: $(sshpass_error $?)" && return "${__BASHIO_EXIT_NOK}" - ) - ) - ) return "${__BASHIO_EXIT_OK}" } @@ -236,12 +232,10 @@ function rsync-folders { fi bashio::log.debug "Syncing ${folders}" - ( - sshpass -p "${REMOTE_PASSWORD}" rsync ${flags} --port ${REMOTE_PORT} --exclude-from='/tmp/rsync_exclude.txt' ${folders} "${rsync_url}/" --delete || ( - bashio::log.error "Error rsyncing folder(s) ${folders} to ${rsync_url}: $(sshpass_error $?)!" + + sshpass -p "${REMOTE_PASSWORD}" rsync ${flags} --port ${REMOTE_PORT} --exclude-from='/tmp/rsync_exclude.txt' ${folders} "${rsync_url}/" --delete || + bashio::log.error "Error rsyncing folder(s) ${folders} to ${rsync_url}: $(sshpass_error $?)!" && return "${__BASHIO_EXIT_NOK}" - ) - ) return "${__BASHIO_EXIT_OK}" } @@ -270,32 +264,27 @@ function rclone-backups { remote_name=$BACKUP_NAME fi bashio::log.info "Copying backup using rclone." - ( - rclone ${DEBUG_FLAG:-} copyto "/backup/${SLUG}.tar" "${rclone_remote_host}:${remote_directory}/${remote_name}.tar" || ( - bashio::log.error "Error rclone ${SLUG}.tar to ${rclone_remote_host}:${remote_directory}/${remote_name}.tar!" + + rclone ${DEBUG_FLAG:-} copyto "/backup/${SLUG}.tar" "${rclone_remote_host}:${remote_directory}/${remote_name}.tar" || + bashio::log.error "Error rclone ${SLUG}.tar to ${rclone_remote_host}:${remote_directory}/${remote_name}.tar!" && return "${__BASHIO_EXIT_NOK}" - ) - ) + fi if bashio::config.true "rclone_sync"; then bashio::log.info "Syncing backups using rclone" - ( - rclone ${DEBUG_FLAG:-} sync "/backup" "${rclone_remote_host}:${remote_directory}" || ( + + rclone ${DEBUG_FLAG:-} sync "/backup" "${rclone_remote_host}:${remote_directory}" || bashio::log.error "Error syncing backups by rclone!" return "${__BASHIO_EXIT_NOK}" - ) - ) fi if bashio::config.true "rclone_restore"; then local restore_name="restore-$(date +%F)" mkdir -p "${restore_name}" bashio::log.info "Restoring backups to ${restore_name} using rclone" - ( - rclone ${DEBUG_FLAG:-} copyto "${rclone_remote_host}:${remote_directory}" "/backup/${restore_name}/" || ( + + rclone ${DEBUG_FLAG:-} copyto "${rclone_remote_host}:${remote_directory}" "/backup/${restore_name}/" || bashio::log.error "Error restoring backups from ${rclone_remote_host}:${remote_directory}!" return "${__BASHIO_EXIT_NOK}" - ) - ) fi return "${__BASHIO_EXIT_OK}" } From c152fffb746e6bbc8dabbb8689905585fc477852 Mon Sep 17 00:00:00 2001 From: Matheson Steplock Date: Fri, 24 May 2024 01:29:23 +0000 Subject: [PATCH 2/5] Update Changelog --- remote-backup/CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/remote-backup/CHANGELOG.md b/remote-backup/CHANGELOG.md index d22bd31..c34968d 100644 --- a/remote-backup/CHANGELOG.md +++ b/remote-backup/CHANGELOG.md @@ -1,3 +1,10 @@ +# 2024.5.0 + +- ⬆️ Update actions/checkout digest to a5ac7e5 by @renovate in https://github.com/ikifar2012/remote-backup-addon/pull/142 +- ⬆️ Update Add-on base image to v16 (major) by @renovate in https://github.com/ikifar2012/remote-backup-addon/pull/144 + +**Full Changelog**: https://github.com/ikifar2012/remote-backup-addon/compare/2024.4.0...2024.5.0 + # 2024.4.0 - ⬆️ Update peter-evans/repository-dispatch action to v3 by @renovate in https://github.com/ikifar2012/remote-backup-addon/pull/130 @@ -72,7 +79,7 @@ **Be aware that some of the configuration options have been renamed and may overwrite your current settings** Please backup your configuration before upgrading by clicking the vertical dots in the top right corner of the add-on configuration page -and click "Edit in YAML", you can then copy that to a text file and map those settings to their new config options as per the +and click "Edit in YAML", you can then copy that to a text file and map those settings to their new config options as per the [documentation](https://addons.mathesonsteplock.ca/docs/addons/remote-backup/basic-config). - enable rsync key-based authentication #51 From 9908afae224d0c552da95e74c1d8d320e60f0e1a Mon Sep 17 00:00:00 2001 From: Matheson Steplock Date: Sat, 1 Jun 2024 04:57:28 +0000 Subject: [PATCH 3/5] Workaround for failed backup alerts in subshells --- remote-backup/run.sh | 102 +++++++++++++++++++++++++++++++------------ 1 file changed, 74 insertions(+), 28 deletions(-) diff --git a/remote-backup/run.sh b/remote-backup/run.sh index be105a0..4e7e1f3 100755 --- a/remote-backup/run.sh +++ b/remote-backup/run.sh @@ -35,6 +35,19 @@ function migrate_config { if bashio::fs.directory_exists "/ssl" && bashio::fs.directory_exists "/config"; then bashio::log.notice "Migration complete." fi + +} +function run_setup { + # create directory for failed backups + bashio::log.notice "Creating directories for failed backup alerts." + mkdir -p /tmp/backup + mkdir -p /tmp/scp + mkdir -p /tmp/rsync + mkdir -p /tmp/rclone + bashio::log.notice "Directories created." + bashio::log.notice "Setup complete." + + } # script global shortcuts declare -r BACKUP_NAME="$(bashio::config 'backup_custom_prefix' '') $(date +'%Y-%m-%d %H-%M')" @@ -98,10 +111,11 @@ function add-ssh-key { # prepare SSH key pair mkdir -p ${SSH_HOME} || bashio::log.error "Failed to create .ssh directory!" if bashio::config.has_value "remote_key"; then + bashio::log.notice "Setting up SSH key pair." ( cp "/config/$(bashio::config 'remote_key')" "${SSH_HOME}/id_rsa" - ssh-keygen -y -f ${SSH_HOME}/id_rsa > ${SSH_HOME}/id_rsa.pub chmod 600 "${SSH_HOME}/id_rsa" + ssh-keygen -y -f ${SSH_HOME}/id_rsa > ${SSH_HOME}/id_rsa.pub chmod 644 "${SSH_HOME}/id_rsa.pub" ) || bashio::log.error "Failed to create SSH key pair!" fi @@ -173,20 +187,24 @@ function create-local-backup { data="$(echo $data | tr -d '}'), \"addons\": ${addons}, \"folders\": ${folders}}" # append addon and folder set if ! SLUG=$(bashio::api.supervisor POST /backups/new/partial "${data}" .slug); then bashio::log.fatal "Error creating ${bak_type} partial backup!" - return "${__BASHIO_EXIT_NOK}" + touch "/tmp/backup/failed" fi else bashio::log.info "Creating ${bak_type} full backup: \"${BACKUP_NAME}\"" if ! SLUG=$(bashio::api.supervisor POST /backups/new/full "${data}" .slug); then bashio::log.fatal "Error creating ${bak_type} full backup!" - return "${__BASHIO_EXIT_NOK}" + touch "/tmp/backup/failed" fi fi - + if bashio::fs.file_exists "/tmp/backup/failed"; then + rm -f "/tmp/backup/failed" + return "${__BASHIO_EXIT_NOK}" + else bashio::log.info "Backup created: ${SLUG}" return "${__BASHIO_EXIT_OK}" + fi } function copy-backup-to-remote { @@ -202,17 +220,28 @@ function copy-backup-to-remote { fi bashio::log.info "Copying backup using SFTP/SCP." + + ( sshpass -p "${REMOTE_PASSWORD}" \ - scp ${DEBUG_FLAG:-} -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || + scp ${DEBUG_FLAG:-} -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || ( bashio::log.warning "SFTP transfer failed, falling back to SCP: $(sshpass_error $?)" sshpass -p "${REMOTE_PASSWORD}" \ - scp ${DEBUG_FLAG:-} -O -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || - bashio::log.error "Error copying backup ${SLUG}.tar to ${remote_directory} on ${REMOTE_HOST}: $(sshpass_error $?)" && - return "${__BASHIO_EXIT_NOK}" - - return "${__BASHIO_EXIT_OK}" + scp ${DEBUG_FLAG:-} -O -F "${SSH_HOME}/config" "/backup/${SLUG}.tar" remote:"${remote_directory}/${remote_name}.tar" || ( + bashio::log.error "Error copying backup ${SLUG}.tar to ${remote_directory} on ${REMOTE_HOST}: $(sshpass_error $?)" + touch "/tmp/scp/failed" + ) + ) + ) + + if bashio::fs.file_exists "/tmp/scp/failed"; then + rm -f "/tmp/scp/failed" + return "${__BASHIO_EXIT_NOK}" + else + return "${__BASHIO_EXIT_OK}" + fi } + function rsync-folders { if ! bashio::config.true "rsync_enabled"; then bashio::log.debug "Rsync disabled." @@ -232,12 +261,18 @@ function rsync-folders { fi bashio::log.debug "Syncing ${folders}" - - sshpass -p "${REMOTE_PASSWORD}" rsync ${flags} --port ${REMOTE_PORT} --exclude-from='/tmp/rsync_exclude.txt' ${folders} "${rsync_url}/" --delete || - bashio::log.error "Error rsyncing folder(s) ${folders} to ${rsync_url}: $(sshpass_error $?)!" && + ( + sshpass -p "${REMOTE_PASSWORD}" rsync ${flags} --port ${REMOTE_PORT} --exclude-from='/tmp/rsync_exclude.txt' ${folders} "${rsync_url}/" --delete || ( + bashio::log.error "Error rsyncing folder(s) ${folders} to ${rsync_url}: $(sshpass_error $?)!" + touch "/tmp/rsync/failed" + ) + ) + if bashio::fs.file_exists "/tmp/rsync/failed"; then + rm -f "/tmp/rsync/failed" return "${__BASHIO_EXIT_NOK}" - - return "${__BASHIO_EXIT_OK}" + else + return "${__BASHIO_EXIT_OK}" + fi } function rclone-backups { @@ -264,29 +299,39 @@ function rclone-backups { remote_name=$BACKUP_NAME fi bashio::log.info "Copying backup using rclone." - - rclone ${DEBUG_FLAG:-} copyto "/backup/${SLUG}.tar" "${rclone_remote_host}:${remote_directory}/${remote_name}.tar" || - bashio::log.error "Error rclone ${SLUG}.tar to ${rclone_remote_host}:${remote_directory}/${remote_name}.tar!" && - return "${__BASHIO_EXIT_NOK}" - + ( + rclone ${DEBUG_FLAG:-} copyto "/backup/${SLUG}.tar" "${rclone_remote_host}:${remote_directory}/${remote_name}.tar" || ( + bashio::log.error "Error rclone ${SLUG}.tar to ${rclone_remote_host}:${remote_directory}/${remote_name}.tar!" + touch "/tmp/rclone/failed" + ) + ) fi if bashio::config.true "rclone_sync"; then bashio::log.info "Syncing backups using rclone" - - rclone ${DEBUG_FLAG:-} sync "/backup" "${rclone_remote_host}:${remote_directory}" || + ( + rclone ${DEBUG_FLAG:-} sync "/backup" "${rclone_remote_host}:${remote_directory}" || ( bashio::log.error "Error syncing backups by rclone!" - return "${__BASHIO_EXIT_NOK}" + touch "/tmp/rclone/failed" + ) + ) fi if bashio::config.true "rclone_restore"; then local restore_name="restore-$(date +%F)" mkdir -p "${restore_name}" bashio::log.info "Restoring backups to ${restore_name} using rclone" - - rclone ${DEBUG_FLAG:-} copyto "${rclone_remote_host}:${remote_directory}" "/backup/${restore_name}/" || + ( + rclone ${DEBUG_FLAG:-} copyto "${rclone_remote_host}:${remote_directory}" "/backup/${restore_name}/" || ( bashio::log.error "Error restoring backups from ${rclone_remote_host}:${remote_directory}!" - return "${__BASHIO_EXIT_NOK}" + touch "/tmp/rclone/failed" + ) + ) + fi + if bashio::fs.file_exists "/tmp/rclone/failed"; then + rm -f "/tmp/rclone/failed" + return "${__BASHIO_EXIT_NOK}" + else + return "${__BASHIO_EXIT_OK}" fi - return "${__BASHIO_EXIT_OK}" } function clone-to-remote { @@ -344,6 +389,7 @@ function delete-local-backup { # general setup and backup set-debug-level migrate_config +run_setup add-ssh-key create-local-backup || die "Local backup process failed! See log for details." clone-to-remote || die "Cloning backup(s) to remote host ${REMOTE_HOST} failed! See log for details." @@ -351,4 +397,4 @@ delete-local-backup || die "Removing local backup(s) failed! See log for details bashio::log.info "Backup process done!" fire-event "ok" "Backup ${BACKUP_NAME} created." -bashio::exit.ok +bashio::exit.ok \ No newline at end of file From 71af8c219ab856f4897cd1aee25a5d664c7ce3c8 Mon Sep 17 00:00:00 2001 From: Matheson Steplock Date: Sat, 1 Jun 2024 05:00:31 +0000 Subject: [PATCH 4/5] Fix error notifications and SSH key permissions --- remote-backup/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/remote-backup/CHANGELOG.md b/remote-backup/CHANGELOG.md index c34968d..946af9b 100644 --- a/remote-backup/CHANGELOG.md +++ b/remote-backup/CHANGELOG.md @@ -1,5 +1,7 @@ # 2024.5.0 +- Fix error notifications +- Fix SSH key permissions - ⬆️ Update actions/checkout digest to a5ac7e5 by @renovate in https://github.com/ikifar2012/remote-backup-addon/pull/142 - ⬆️ Update Add-on base image to v16 (major) by @renovate in https://github.com/ikifar2012/remote-backup-addon/pull/144 From c44efc664e10d5b597abffa3b6aa524810cb04a5 Mon Sep 17 00:00:00 2001 From: Matheson Steplock Date: Sat, 1 Jun 2024 05:01:36 +0000 Subject: [PATCH 5/5] Update version to 2024.6.0 --- remote-backup/CHANGELOG.md | 2 +- remote-backup/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/remote-backup/CHANGELOG.md b/remote-backup/CHANGELOG.md index 946af9b..a3720f3 100644 --- a/remote-backup/CHANGELOG.md +++ b/remote-backup/CHANGELOG.md @@ -1,4 +1,4 @@ -# 2024.5.0 +# 2024.6.0 - Fix error notifications - Fix SSH key permissions diff --git a/remote-backup/config.yaml b/remote-backup/config.yaml index 6bf3e2d..c600e3c 100644 --- a/remote-backup/config.yaml +++ b/remote-backup/config.yaml @@ -1,5 +1,5 @@ name: Remote Backup -version: "2024.4.0" +version: "2024.6.0" slug: remote_backup description: Automatically create and transfer HA backups using SFTP (SCP), rsync, or rclone (experimental) image: ikifar/remote-backup-{arch}