From 463c963be2be0f4a711ab09401ec6f964965ae9e Mon Sep 17 00:00:00 2001 From: Robin Meier Date: Wed, 21 Aug 2024 21:02:45 +0200 Subject: [PATCH] Update backup scripts --- backup.sh | 103 ++++++++++++++++++++------ backup_borg.sh | 106 +++++++++++++++++++++++++++ backup_external_disk.sh | 157 ---------------------------------------- backup_remote.sh | 156 --------------------------------------- config/backup.EXAMPLE | 18 +++-- 5 files changed, 200 insertions(+), 340 deletions(-) create mode 100755 backup_borg.sh delete mode 100755 backup_external_disk.sh delete mode 100755 backup_remote.sh diff --git a/backup.sh b/backup.sh index 5747e3d..2f35340 100755 --- a/backup.sh +++ b/backup.sh @@ -15,8 +15,6 @@ # Author: Robin Meier - robin@meier.si ################################################################################ -# TODO: Check if borg installed - # Load configuration script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) set -o allexport @@ -28,36 +26,97 @@ logfile=${script_dir}/log/backup.log log_identifier="BACKUP" source ${script_dir}/functions/logging.sh +# Check if executables installed +EXEC_DEPS="borg grep sed zfs tac mount umount" +if ! which $EXEC_DEPS >/dev/null 2>&1; then + log_echo "Missing executables! ($EXEC_DEPS)" >&2 + exit 9 +fi + log "Backup Script Started" -# Run backup preparations +# Run backup preparations (scripts for db dumps etc.) ${script_dir}/backup_preparations.sh # Check preparations exit code prep_exit=$? if [[ $prep_exit > 0 ]]; then log_echo "Preparations failed with exit code $prep_exit" + echo -n "Preparations failed with exit code $prep_exit" | ${script_dir}/helpers/tg_notify.sh + + # Do not continue with the backup exit $prep_exit fi -# Run external disk backup if set -external_exit=0 -if [[ -n $EXTERNAL_DISK_BACKUP_ENV_FILE ]]; then - ${script_dir}/backup_external_disk.sh &>> $logfile - external_exit=$? -fi -if [[ $external_exit -gt 0 ]]; then - log_echo "External disk backup failed with exit code $external_exit" -fi +SNAPSHOT_TIMESTAMP=$(date +%Y%m%d%H%M%S) # Make sure this is digits only or adapt regex below -# Run remote disk backup if set -remote_exit=0 -if [[ -n $REMOTE_BACKUP_ENV_FILE ]]; then - ${script_dir}/backup_remote.sh &>> $logfile - remote_exit=$? -fi -if [[ $remote_exit -gt 0 ]]; then - log_echo "External disk backup failed with exit code $remote_exit" -fi +# Create ZFS Snapshots +for DS in $BORG_ZFS_DATASETS; do + zfs snapshot -r "$DS@$SNAPSHOT_TIMESTAMP" +done -log "Backup Script Finished" +# Mount snapshots under $ZFS_SNAPSHOT_MOUNTPOINT +for SNAP in $(zfs list -t snapshot -o name | grep "$SNAPSHOT_TIMESTAMP"); do + SNAP_MOUNTPOINT="$ZFS_SNAPSHOT_MOUNTPOINT/$(sed 's/@[[:digit:]]\+$//' <<< "$SNAP")" + log "Mount $SNAP at $SNAP_MOUNTPOINT" + mkdir -p "$SNAP_MOUNTPOINT" + mount -t zfs "$SNAP" "$SNAP_MOUNTPOINT" +done + +# Run backup script for each configuration +global_exit=0 +for BORG_ENV in $BORG_DESTINATION_ENV_FILES; do + log "BORG Backup for $BORG_ENV" + if [[ ! -f "$BORG_ENV" ]]; then + # Exit if given env file does not exist + exit 8 + fi + set -o allexport + source $BORG_ENV + set +o allexport + + log "Backup debian system..." + ${script_dir}/backup_borg.sh debian $(date +%Y%m%d%H%M%S) "$BORG_DEBIAN_DIRS" "--exclude=var/tmp/* --exclude=var/lib/docker" &>> $logfile + system_exit=$? + log "Backupped debian system (exit code: $system_exit)" + if [[ $system_exit -gt 1 ]]; then + log_echo "System backup failed with error code $system_exit" + echo -n "System backup failed with error code $system_exit" | ${script_dir}/helpers/tg_notify.sh + fi + + log "Backup zfs array" + ${script_dir}/backup_borg.sh ${ZFS_ARRAY_NAME:-zfs} $SNAPSHOT_TIMESTAMP "$ZFS_SNAPSHOT_MOUNTPOINT" "" &>> $logfile + zfs_exit=$? + log "Backupped zfs array (exit code: $zfs_exit)" + if [[ $zfs_exit -gt 0 ]]; then + log_echo "ZFS Array Backup Failed with Warning or Error (Exit Code: $external_exit)" + echo -n "ZFS Array Backup Failed with Warning or Error (Exit Code: $external_exit)" | ${script_dir}/helpers/tg_notify.sh + fi + + borg_env_exit=$(( system_exit > zfs_exit ? system_exit : zfs_exit )) + global_exit=$(( borg_env_exit > global_exit ? borg_env_exit : global_exit )) +done + +# # Wait file for inspecting of /mnt/snapshot +# touch /mnt/snapshot/wait_file +# while : ; do +# [[ ! -f "/mnt/snapshot/wait_file" ]] && break +# log_echo "Waiting.." +# sleep 2 +# done + +# Unmount snapshots in reverse order (tac reverses lines) +for SNAP in $(zfs list -t snapshot -o name | grep "$SNAPSHOT_TIMESTAMP" | tac); do + log "Unmount $SNAP" + umount "$SNAP" +done + +# Destroy snapshots (recursive) for each dataset +for DS in $BORG_ZFS_DATASETS; do + log "Destroy snapshot $DS" + zfs destroy -r "$DS@$SNAPSHOT_TIMESTAMP" +done + +log "Backup Script Finished ($global_exit)" + +exit $global_exit diff --git a/backup_borg.sh b/backup_borg.sh new file mode 100755 index 0000000..83ff552 --- /dev/null +++ b/backup_borg.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +################################################################################ +# Backup Borg +# --------------------------- +# +# TODO +# +# Author: Robin Meier - robin@meier.si +################################################################################ + +# Import logging functionality +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +logfile=${script_dir}/log/backup.log +log_identifier="BORG" +source ${script_dir}/functions/logging.sh + +# Input parameters +BORG_NAME=$1 +TIMESTAMP=$2 +BORG_DIRS=$3 +BORG_OPTS=$4 +if [[ -z "$BORG_NAME" || -z "$TIMESTAMP" || -z "$BORG_DIRS" ]]; then + log_echo "[ERROR] Positional parameter missing! (name, timestamp, directories (, borg options))" + exit 40 +fi + +# Check if required env vars are set +if [[ -z "$BORG_REPO" || -z "$BORG_PASSPHRASE" ]]; then + log_echo "[ERROR] Repo location or passphrase env var missing!" + exit 42 +fi + +# Automated negative response to these questions +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no + +# Error handling +# trap 'log "Backup interrupted"; exit 2' INT TERM # untested +trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM + +# Output current version +log_echo "Backup borg version: $(borg --version)" + +# Create backup +log_echo "Creating backup..." + +borg create $BORG_BACKUP_OPTS $BORG_OPTS \ + --stats \ + --verbose \ + --filter AME \ + --list \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + ::"{hostname}-${BORG_NAME}-${TIMESTAMP}" \ + $BORG_DIRS + +backup_exit=$? + +log_echo "Finished creating backup ($backup_exit)" + + +# Prune no longer needed backups +log "Pruning backups from repository" + +borg prune \ + --list \ + --glob-archives "{hostname}-${BORG_NAME}-*" \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 + +prune_exit=$? + +log "Finished pruning backups from repository ($prune_exit)" + + +# Compact repository +log "Compacting repository" + +# TODO: Maybe first check if compacting is supported by the client? (Older versions) +borg compact + +compact_exit=$? + +log "Finished compacting repository ($compact_exit)" + + +# Calculate global exit code +global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) +global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) + +if [ ${global_exit} -eq 0 ]; then + log "Backup, Prune, and Compact finished successfully" +elif [ ${global_exit} -eq 1 ]; then + log_echo "Backup, Prune, and/or Compact finished with warnings" +else + log_echo "Backup, Prune, and/or Compact finished with errors" +fi + + +log "Backup Finished ($global_exit)" + +exit ${global_exit} diff --git a/backup_external_disk.sh b/backup_external_disk.sh deleted file mode 100755 index a314e62..0000000 --- a/backup_external_disk.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash - -################################################################################ -# Backup External Disk Script -# --------------------------- -# -# THIS SCRIPT IS WORK IN PROGRESS AND WILL PROBABLY BE REPLACED SOON! -# -# Author: Robin Meier - robin@meier.si -################################################################################ - -# Load configuration -script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -set -o allexport -source ${script_dir}/config/backup > /dev/null 2>&1 -set +o allexport - -# Import logging functionality -logfile=${script_dir}/log/backup.log -log_identifier="EXTERN" -source ${script_dir}/functions/logging.sh - -# Exit if no env file is given -if [[ -z "$EXTERNAL_DISK_BACKUP_ENV_FILE" ]]; then - exit 0 -fi - -set -o allexport -source $EXTERNAL_DISK_BACKUP_ENV_FILE -set +o allexport - -log "External Disk Backup Started" - -# Check if required env vars are set -if [[ -z "$BORG_REPO" || -z "$BORG_PASSPHRASE" || -z "$BORG_DEBIAN_DIRS" || -z "$BORG_ZFS_DIRS" ]]; then - log "[ERROR] Required env var missing in $EXTERNAL_DISK_BACKUP_ENV_FILE" - exit 42 -fi - -if [[ -z "$ZFS_ARRAY_NAME" ]]; then - ZFS_ARRAY_NAME=zfs -fi - -# Automated negative response to these questions -export BORG_RELOCATED_REPO_ACCESS_IS_OK=no -export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no - -# Error handling -trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM - -# Output current version -borg --version - - -# Backup debian -log "Creating debian backup" - -borg create $BORG_OPTS \ - --verbose \ - --filter AME \ - --list \ - --show-rc \ - --compression lz4 \ - --one-file-system \ - --exclude-caches \ - --exclude 'root/.cache' \ - --exclude 'home/*/.cache/*' \ - --exclude 'var/tmp/*' \ - --exclude 'var/lib/docker/*' \ - ::'{hostname}-debian-{now}' \ - $BORG_DEBIAN_DIRS - -backup_exit_1=$? - -log "Finished creating debian backup ($backup_exit_1)" - - -# Backup zfs array -log "Creating $ZFS_ARRAY_NAME backup" - -borg create $BORG_OPTS \ - --verbose \ - --filter AME \ - --list \ - --show-rc \ - --compression lz4 \ - --exclude-caches \ - ::"{hostname}-${ZFS_ARRAY_NAME}-{now}" \ - $BORG_ZFS_DIRS - -backup_exit_2=$? - -log "Finished creating $ZFS_ARRAY_NAME backup ($backup_exit_2)" - -backup_exit=$(( backup_exit_2 > backup_exit_1 ? backup_exit_2 : backup_exit_1 )) - - -# Prune no longer needed backups -log "Pruning debian backups from repository" - -borg prune \ - --list \ - --glob-archives '{hostname}-debian-*' \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_1=$? - -log "Finished pruning debian backups from repository ($prune_exit_1)" - -log "Pruning $ZFS_ARRAY_NAME backups from repository" - -borg prune \ - --list \ - --glob-archives "{hostname}-${ZFS_ARRAY_NAME}-*" \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_2=$? - -log "Finished pruning $ZFS_ARRAY_NAME backups from repository ($prune_exit_2)" - -prune_exit=$(( prune_exit_2 > prune_exit_1 ? prune_exit_2 : prune_exit_1 )) - - -# Compact repository -log "Compacting repository" - -# TODO: Maybe first check if compacting is supported by the client? (Older versions) - -borg compact - -compact_exit=$? - -log "Finished compacting repository ($compact_exit)" - - -# Calculate global exit code -global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) -global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) - -if [ ${global_exit} -eq 0 ]; then - log "Backup, Prune, and Compact finished successfully" -elif [ ${global_exit} -eq 1 ]; then - log_echo "Backup, Prune, and/or Compact finished with warnings" -else - log_echo "Backup, Prune, and/or Compact finished with errors" -fi - - -log "External Disk Backup Finished" - -exit ${global_exit} diff --git a/backup_remote.sh b/backup_remote.sh deleted file mode 100755 index 0b44311..0000000 --- a/backup_remote.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -################################################################################ -# Backup Remote Script -# -------------------- -# -# THIS SCRIPT IS WORK IN PROGRESS AND WILL PROBABLY BE REPLACED SOON! -# -# Author: Robin Meier - robin@meier.si -################################################################################ - -script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -set -o allexport -source ${script_dir}/config/backup > /dev/null 2>&1 -set +o allexport - -# Import logging functionality -logfile=${script_dir}/log/backup.log -log_identifier="REMOTE" -source ${script_dir}/functions/logging.sh - -# Exit if no env file is given -if [[ -z "$REMOTE_BACKUP_ENV_FILE" ]]; then - exit 0 -fi - -set -o allexport -source $REMOTE_BACKUP_ENV_FILE -set +o allexport - -log "Remote Backup Started" - -# Check if required env vars are set -if [[ -z "$BORG_REPO" || -z "$BORG_PASSPHRASE" || -z "$BORG_DEBIAN_DIRS" || -z "$BORG_ZFS_DIRS" ]]; then - log "[ERROR] Required env var missing in $EXTERNAL_DISK_BACKUP_ENV_FILE" - exit 42 -fi - -if [[ -z "$ZFS_ARRAY_NAME" ]]; then - ZFS_ARRAY_NAME=zfs -fi - -# Automated negative response to these questions -export BORG_RELOCATED_REPO_ACCESS_IS_OK=no -export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no - -# Error handling -trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM - -# Output current version -borg --version - - -# Backup debian -log "Creating debian backup" - -borg create $BORG_OPTS \ - --verbose \ - --filter AME \ - --list \ - --show-rc \ - --compression lz4 \ - --one-file-system \ - --exclude-caches \ - --exclude 'root/.cache' \ - --exclude 'home/*/.cache/*' \ - --exclude 'var/tmp/*' \ - --exclude 'var/lib/docker/*' \ - ::'{hostname}-debian-{now}' \ - $BORG_DEBIAN_DIRS - -backup_exit_1=$? - -log "Finished creating debian backup ($backup_exit_1)" - - -# Backup zfs array -log "Creating $ZFS_ARRAY_NAME backup" - -borg create $BORG_OPTS \ - --verbose \ - --filter AME \ - --list \ - --show-rc \ - --compression lz4 \ - --exclude-caches \ - ::"{hostname}-${ZFS_ARRAY_NAME}-{now}" \ - $BORG_ZFS_DIRS - -backup_exit_2=$? - -log "Finished creating $ZFS_ARRAY_NAME backup ($backup_exit_2)" - -backup_exit=$(( backup_exit_2 > backup_exit_1 ? backup_exit_2 : backup_exit_1 )) - - -# Prune no longer needed backups -log "Pruning debian backups from repository" - -borg prune \ - --list \ - --glob-archives '{hostname}-debian-*' \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_1=$? - -log "Finished pruning debian backups from repository ($prune_exit_1)" - -log "Pruning $ZFS_ARRAY_NAME backups from repository" - -borg prune \ - --list \ - --glob-archives "{hostname}-${ZFS_ARRAY_NAME}-*" \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_2=$? - -log "Finished pruning $ZFS_ARRAY_NAME backups from repository ($prune_exit_2)" - -prune_exit=$(( prune_exit_2 > prune_exit_1 ? prune_exit_2 : prune_exit_1 )) - - -# Compact repository -log "Compacting repository" - -# TODO: Maybe first check if compacting is supported by the client? (Older versions) - -borg compact - -compact_exit=$? - -log "Finished compacting repository ($compact_exit)" - - -# Calculate global exit code -global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) -global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) - -if [ ${global_exit} -eq 0 ]; then - log "Backup, Prune, and Compact finished successfully" -elif [ ${global_exit} -eq 1 ]; then - log_echo "Backup, Prune, and/or Compact finished with warnings" -else - log_echo "Backup, Prune, and/or Compact finished with errors" -fi - - -log "Remote Backup Finished" - -exit ${global_exit} diff --git a/config/backup.EXAMPLE b/config/backup.EXAMPLE index 121831b..d8fd892 100644 --- a/config/backup.EXAMPLE +++ b/config/backup.EXAMPLE @@ -1,11 +1,19 @@ +# Recursive searched directories for pre backup scripts PRE_BACKUP_SCRIPT_DIRS="/root /mnt/trident/docker" + +# Name of backup of zfs array (optional, fallback: zfs) ZFS_ARRAY_NAME=trident -EXTERNAL_DISK_BACKUP_ENV_FILE=/root/.borgenv-wdexternal -REMOTE_BACKUP_ENV_FILE= -#REMOTE_BACKUP_ENV_FILE=/root/.borgenv-qbli +# Borg Backup Destination ENV Files +BORG_DESTINATION_ENV_FILES="/root/.borgenv-wdexternal /root/.borgenv-qbli" -BORG_OPTS="--stats --compression lz4 --checkpoint-interval 86400" +# General options for borg backup (TODO: Unclear what this is for) +BORG_BACKUP_OPTS="" BORG_DEBIAN_DIRS="/home /root /etc /var" -BORG_ZFS_DIRS="/mnt/trident/docker /mnt/trident/private" + +# ZFS Datasets to be backed up +BORG_ZFS_DATASETS="trident/docker trident/private" + +# Mountpoint for zfs dataset snapshot +ZFS_SNAPSHOT_MOUNTPOINT="/mnt/snapshot"