From a7c183c601064eae18063f1a659a043746dda769 Mon Sep 17 00:00:00 2001 From: Robin Meier Date: Tue, 14 May 2024 19:05:32 +0200 Subject: [PATCH] Add first version of backup scripts (WIP) --- README.md | 4 + backup.sh | 63 ++++++++++++++++ backup_external_disk.sh | 157 ++++++++++++++++++++++++++++++++++++++++ backup_preparations.sh | 51 +++++++++++++ backup_remote.sh | 156 +++++++++++++++++++++++++++++++++++++++ config/backup.EXAMPLE | 11 +++ system_backup.sh | 97 ------------------------- 7 files changed, 442 insertions(+), 97 deletions(-) create mode 100755 backup.sh create mode 100755 backup_external_disk.sh create mode 100755 backup_preparations.sh create mode 100755 backup_remote.sh create mode 100644 config/backup.EXAMPLE delete mode 100755 system_backup.sh diff --git a/README.md b/README.md index 9a3a69a..6801379 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,10 @@ radioelephant ALL=NOPASSWD: /usr/bin/zfs mount -a -l radioelephant ALL=(docker) NOPASSWD: /usr/bin/docker ``` +### Backups + +*Backup Scipts are not documented at this moment, see source code for more information. Expect changes as this is work in progress.* + ## Contributors - Robin Meier (robin@meier.si) diff --git a/backup.sh b/backup.sh new file mode 100755 index 0000000..96ce4cf --- /dev/null +++ b/backup.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +################################################################################ +# Backup Script +# ------------- +# +# This script orchestrates and performs system backups with borg according to +# the environment variables in config/backup and .borgenv files per borg +# repository you wish to backup to. This script assumes two backups for each +# repository, splitting up the base operating system (debian) and the zfs array +# (zfs) which can be custom named in the config file. +# +# EXPECT THESE BACKUP SCRIPTS TO CHANGE IN THE NEAR FUTURE! +# +# Author: Robin Meier - robin@meier.si +################################################################################ + +# TODO: Check if borg installed + +# Load configuration +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -o allexport +source ${script_dir}/config/backup > /dev/null 2>&1 +set +o allexport + +# Import logging functionality +logfile=${script_dir}/log/backup.log +log_identifier="BACKUP" +source ${script_dir}/functions/logging.sh + +log "Backup Script Started" + +# Run backup preparations +${script_dir}/backup_preparations.sh + +# Check preparations exit code +prep_exit=$? +if [[ $prep_exit > 0 ]]; then + log_echo "Preparations failed with exit code $prep_exit" + exit $prep_exit +fi + +# Run external disk backup if set +external_exit=0 +if [[ -n $EXTERNAL_DISK_BACKUP_ENV_FILE ]]; then + ${script_dir}/backup_external_disk.sh &>> $logfile + external_exit=$? +fi +if [[ $external_exit -gt 0 ]]; then + log_echo "External disk backup failed with exit code $external_exit" +fi + +# Run remote disk backup if set +remote_exit=0 +if [[ -n $EXTERNAL_DISK_BACKUP_ENV_FILE ]]; then + ${script_dir}/backup_remote.sh &>> $logfile + remote_exit=$? +fi +if [[ $remote_exit -gt 0 ]]; then + log_echo "External disk backup failed with exit code $remote_exit" +fi + +log "Backup Script Finished" diff --git a/backup_external_disk.sh b/backup_external_disk.sh new file mode 100755 index 0000000..a314e62 --- /dev/null +++ b/backup_external_disk.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +################################################################################ +# Backup External Disk Script +# --------------------------- +# +# THIS SCRIPT IS WORK IN PROGRESS AND WILL PROBABLY BE REPLACED SOON! +# +# Author: Robin Meier - robin@meier.si +################################################################################ + +# Load configuration +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -o allexport +source ${script_dir}/config/backup > /dev/null 2>&1 +set +o allexport + +# Import logging functionality +logfile=${script_dir}/log/backup.log +log_identifier="EXTERN" +source ${script_dir}/functions/logging.sh + +# Exit if no env file is given +if [[ -z "$EXTERNAL_DISK_BACKUP_ENV_FILE" ]]; then + exit 0 +fi + +set -o allexport +source $EXTERNAL_DISK_BACKUP_ENV_FILE +set +o allexport + +log "External Disk Backup Started" + +# Check if required env vars are set +if [[ -z "$BORG_REPO" || -z "$BORG_PASSPHRASE" || -z "$BORG_DEBIAN_DIRS" || -z "$BORG_ZFS_DIRS" ]]; then + log "[ERROR] Required env var missing in $EXTERNAL_DISK_BACKUP_ENV_FILE" + exit 42 +fi + +if [[ -z "$ZFS_ARRAY_NAME" ]]; then + ZFS_ARRAY_NAME=zfs +fi + +# Automated negative response to these questions +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no + +# Error handling +trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM + +# Output current version +borg --version + + +# Backup debian +log "Creating debian backup" + +borg create $BORG_OPTS \ + --verbose \ + --filter AME \ + --list \ + --show-rc \ + --compression lz4 \ + --one-file-system \ + --exclude-caches \ + --exclude 'root/.cache' \ + --exclude 'home/*/.cache/*' \ + --exclude 'var/tmp/*' \ + --exclude 'var/lib/docker/*' \ + ::'{hostname}-debian-{now}' \ + $BORG_DEBIAN_DIRS + +backup_exit_1=$? + +log "Finished creating debian backup ($backup_exit_1)" + + +# Backup zfs array +log "Creating $ZFS_ARRAY_NAME backup" + +borg create $BORG_OPTS \ + --verbose \ + --filter AME \ + --list \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + ::"{hostname}-${ZFS_ARRAY_NAME}-{now}" \ + $BORG_ZFS_DIRS + +backup_exit_2=$? + +log "Finished creating $ZFS_ARRAY_NAME backup ($backup_exit_2)" + +backup_exit=$(( backup_exit_2 > backup_exit_1 ? backup_exit_2 : backup_exit_1 )) + + +# Prune no longer needed backups +log "Pruning debian backups from repository" + +borg prune \ + --list \ + --glob-archives '{hostname}-debian-*' \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 + +prune_exit_1=$? + +log "Finished pruning debian backups from repository ($prune_exit_1)" + +log "Pruning $ZFS_ARRAY_NAME backups from repository" + +borg prune \ + --list \ + --glob-archives "{hostname}-${ZFS_ARRAY_NAME}-*" \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 + +prune_exit_2=$? + +log "Finished pruning $ZFS_ARRAY_NAME backups from repository ($prune_exit_2)" + +prune_exit=$(( prune_exit_2 > prune_exit_1 ? prune_exit_2 : prune_exit_1 )) + + +# Compact repository +log "Compacting repository" + +# TODO: Maybe first check if compacting is supported by the client? (Older versions) + +borg compact + +compact_exit=$? + +log "Finished compacting repository ($compact_exit)" + + +# Calculate global exit code +global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) +global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) + +if [ ${global_exit} -eq 0 ]; then + log "Backup, Prune, and Compact finished successfully" +elif [ ${global_exit} -eq 1 ]; then + log_echo "Backup, Prune, and/or Compact finished with warnings" +else + log_echo "Backup, Prune, and/or Compact finished with errors" +fi + + +log "External Disk Backup Finished" + +exit ${global_exit} diff --git a/backup_preparations.sh b/backup_preparations.sh new file mode 100755 index 0000000..0a7afac --- /dev/null +++ b/backup_preparations.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +################################################################################ +# Backup Preparations Script +# -------------------------- +# +# This script should be run before performing a backup as it will do some +# preparations. This script searches in all directories in +# PRE_BACKUP_SCRIPT_DIRS for all executable scripts called +# `pre_borg_backup.sh`. Every found script is executed and should do any +# preparing action for a full system backup, such as dumping a database of a +# docker container. +# +# Author: Robin Meier - robin@meier.si +################################################################################ + +# Load configuration +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -o allexport +source ${script_dir}/config/backup > /dev/null 2>&1 +set +o allexport + +# Import logging functionality +logfile=${script_dir}/log/backup.log +log_identifier="PREPAR" +source ${script_dir}/functions/logging.sh + +# Exit if no backup script dirs are found +if [[ -z "$PRE_BACKUP_SCRIPT_DIRS" ]]; then + exit 0 +fi + +highest_exit=0 + +log "Preparations Started" + +for directory in $PRE_BACKUP_SCRIPT_DIRS; do + for script in $(find $directory -name pre_borg_backup.sh -executable); do + log "$(date +%Y%m%d_%H%M%S) Running script $script" + + $script >> ${script_dir}/log/backup.log 2>&1 + + script_exit=$? + + highest_exit=$(( script_exit > highest_exit ? script_exit : highest_exit )) + done +done + +log "Preparations Finished" + +exit $highest_exit diff --git a/backup_remote.sh b/backup_remote.sh new file mode 100755 index 0000000..f78fde5 --- /dev/null +++ b/backup_remote.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +################################################################################ +# Backup Remote Script +# -------------------- +# +# THIS SCRIPT IS WORK IN PROGRESS AND WILL PROBABLY BE REPLACED SOON! +# +# Author: Robin Meier - robin@meier.si +################################################################################ + +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -o allexport +source ${script_dir}/config/backup > /dev/null 2>&1 +set +o allexport + +# Import logging functionality +logfile=${script_dir}/log/backup.log +log_identifier="REMOTE" +source ${script_dir}/functions/logging.sh + +# Exit if no env file is given +if [[ -z "$REMOTE_BACKUP_ENV_FILE" ]]; then + exit 0 +fi + +set -o allexport +source $EXTERNAL_DISK_BACKUP_ENV_FILE +set +o allexport + +log "Remote Backup Started" + +# Check if required env vars are set +if [[ -z "$BORG_REPO" || -z "$BORG_PASSPHRASE" || -z "$BORG_DEBIAN_DIRS" || -z "$BORG_ZFS_DIRS" ]]; then + log "[ERROR] Required env var missing in $EXTERNAL_DISK_BACKUP_ENV_FILE" + exit 42 +fi + +if [[ -z "$ZFS_ARRAY_NAME" ]]; then + ZFS_ARRAY_NAME=zfs +fi + +# Automated negative response to these questions +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no + +# Error handling +trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM + +# Output current version +borg --version + + +# Backup debian +log "Creating debian backup" + +borg create $BORG_OPTS \ + --verbose \ + --filter AME \ + --list \ + --show-rc \ + --compression lz4 \ + --one-file-system \ + --exclude-caches \ + --exclude 'root/.cache' \ + --exclude 'home/*/.cache/*' \ + --exclude 'var/tmp/*' \ + --exclude 'var/lib/docker/*' \ + ::'{hostname}-debian-{now}' \ + $BORG_DEBIAN_DIRS + +backup_exit_1=$? + +log "Finished creating debian backup ($backup_exit_1)" + + +# Backup zfs array +log "Creating $ZFS_ARRAY_NAME backup" + +borg create $BORG_OPTS \ + --verbose \ + --filter AME \ + --list \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + ::"{hostname}-${ZFS_ARRAY_NAME}-{now}" \ + $BORG_ZFS_DIRS + +backup_exit_2=$? + +log "Finished creating $ZFS_ARRAY_NAME backup ($backup_exit_2)" + +backup_exit=$(( backup_exit_2 > backup_exit_1 ? backup_exit_2 : backup_exit_1 )) + + +# Prune no longer needed backups +log "Pruning debian backups from repository" + +borg prune \ + --list \ + --glob-archives '{hostname}-debian-*' \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 + +prune_exit_1=$? + +log "Finished pruning debian backups from repository ($prune_exit_1)" + +log "Pruning $ZFS_ARRAY_NAME backups from repository" + +borg prune \ + --list \ + --glob-archives "{hostname}-${ZFS_ARRAY_NAME}-*" \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 + +prune_exit_2=$? + +log "Finished pruning $ZFS_ARRAY_NAME backups from repository ($prune_exit_2)" + +prune_exit=$(( prune_exit_2 > prune_exit_1 ? prune_exit_2 : prune_exit_1 )) + + +# Compact repository +log "Compacting repository" + +# TODO: Maybe first check if compacting is supported by the client? (Older versions) + +borg compact + +compact_exit=$? + +log "Finished compacting repository ($compact_exit)" + + +# Calculate global exit code +global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) +global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) + +if [ ${global_exit} -eq 0 ]; then + log "Backup, Prune, and Compact finished successfully" +elif [ ${global_exit} -eq 1 ]; then + log_echo "Backup, Prune, and/or Compact finished with warnings" +else + log_echo "Backup, Prune, and/or Compact finished with errors" +fi + + +log "Remote Backup Finished" + +exit ${global_exit} diff --git a/config/backup.EXAMPLE b/config/backup.EXAMPLE new file mode 100644 index 0000000..121831b --- /dev/null +++ b/config/backup.EXAMPLE @@ -0,0 +1,11 @@ +PRE_BACKUP_SCRIPT_DIRS="/root /mnt/trident/docker" +ZFS_ARRAY_NAME=trident + +EXTERNAL_DISK_BACKUP_ENV_FILE=/root/.borgenv-wdexternal +REMOTE_BACKUP_ENV_FILE= +#REMOTE_BACKUP_ENV_FILE=/root/.borgenv-qbli + +BORG_OPTS="--stats --compression lz4 --checkpoint-interval 86400" + +BORG_DEBIAN_DIRS="/home /root /etc /var" +BORG_ZFS_DIRS="/mnt/trident/docker /mnt/trident/private" diff --git a/system_backup.sh b/system_backup.sh deleted file mode 100755 index 32648d6..0000000 --- a/system_backup.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -# -# Script configuration -# - -script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -set -o allexport -source ${script_dir}/config/system_backup -set +o allexport - -export BORG_REPO=$MOUNTPOINT/borg/backups - -DATE=$(date -u +%Y-%m-%d-%H-%M) - - -# -# Create backups -# - -# Options for borg create -BORG_OPTS="--stats --compression lz4 --one-file-system --checkpoint-interval 86400" - -# No one can answer if Borg asks these questions, it is better to just fail quickly -# instead of hanging. -export BORG_RELOCATED_REPO_ACCESS_IS_OK=no -export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no - -# some helpers and error handling: -info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; } -trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM - -borg --version - -info "Starting backup for $DATE" - -borg create $BORG_OPTS \ - --verbose \ - --filter AME \ - --list \ - --stats \ - --show-rc \ - --compression lz4 \ - --exclude-caches \ - --exclude 'root/.cache' \ - --exclude 'home/*/.cache/*' \ - --exclude 'var/tmp/*' \ - ::'{hostname}-debian-{now}' \ - /home /root /etc /var - -backup_exit=$? - - -info "Pruning repository" - -borg prune \ - --list \ - --glob-archives '{hostname}-debian-*' \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_1=$? - -borg prune \ - --list \ - --glob-archives '{hostname}-trident-*' \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 6 - -prune_exit_2=$? -prune_exit=$(( prune_exit_2 > prune_exit_1 ? prune_exit_2 : prune_exit_1 )) - - -info "Compacting repository" - -borg compact - -compact_exit=$? - - -# use highest exit code as global exit code -global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) -global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit )) - -if [ ${global_exit} -eq 0 ]; then - info "Backup, Prune, and Compact finished successfully" -elif [ ${global_exit} -eq 1 ]; then - info "Backup, Prune, and/or Compact finished with warnings" -else - info "Backup, Prune, and/or Compact finished with errors" -fi - -exit ${global_exit}