]> arthur.barton.de Git - backup-script.git/blobdiff - bin/backup-script
Fix and mark several notices & warnings of shellcheck(1)
[backup-script.git] / bin / backup-script
index 24d3d9f107ab0cdb8fb38c0b1b2a8dc088572caa..bd329411ae4d709c79a924ae698d70ab35245615 100755 (executable)
@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # backup-script system for cloning systems using rsync
-# Copyright (c)2008-2013 Alexander Barton, alex@barton.de
+# Copyright (c)2008-2016 Alexander Barton <alex@barton.de>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # Please read the file COPYING, README and AUTHORS for more information.
 #
 
-NAME=`basename $0`
-CONF_D="/etc/backup-script.d"
+NAME=$(basename "$0")
 PIDFILE="/var/run/$NAME.pid"
 
 DRYRUN=0
 VERBOSE=0
+TAG=""
 
 export LC_ALL=C
 
@@ -23,43 +23,48 @@ declare -i count_all=0
 declare -i count_started=0
 declare -i count_ok=0
 declare -i count_ok_vanished=0
+declare -i count_enabled=0
 
 destinations=""
 
 # Default settings, can be overwritten in backup-script.conf:
+[ -d "/usr/local/etc/backup-script.d" ] \
+       && conf_d="/usr/local/etc/backup-script.d" \
+       || conf_d="/etc/backup-script.d"
+setup_exec=""
 pre_exec=""
 post_exec=""
-default_target=""
+default_backup_type="rsync"
+default_source_root="/"
+default_files="running-config"
+default_target="/var/backups"
 default_user="root"
 default_ssh_args_add=""
 default_rsync_args_add=""
 default_exclude_args_add=""
+default_exclude_dirs_add=""
 default_compress=1
 default_ping=1
 default_local=0
+default_generations=0
+default_io_timeout="1800"
+default_job_pre_exec=""
+default_job_post_exec=""
+default_tags=""
 
 Usage() {
        echo "Usage: $NAME [<options>] [<system> [<system> [...]]]"
        echo
-       echo "  -p, --progress    Show progress, see rsync(1)."
-       echo "  -d, --dry-run     Test run only, don't copy any data."
+       echo "  -n, --dry-run       Test run only, don't copy any data."
+       echo "  -p, --progress      Show progress, see rsync(1)."
+       echo "  -t TAG, --tag TAG   Only run jobs with tag TAG."
        echo
        echo "When no <system> is given, all defined systems are used."
        echo
-       exit 1
-}
-
-Log() {
-       logger -t "$NAME" "$*"
-}
-
-Message() {
-       echo "$*"
-}
-
-MessageLog() {
-       Log "$*"
-       Message "$*"
+       # shellcheck disable=SC2086
+       echo -e $config_info
+       echo
+       exit 2
 }
 
 CleanUp() {
@@ -81,9 +86,213 @@ GotSignal() {
        CleanUp
        echo -n "Aborted: "; date
        echo
+       sleep 3
        exit 9
 }
 
+ExecJob() {
+       local what="$1"
+       local cmd="$2"
+
+       echo "Running job ${what}-exec command ..."
+       [ "$local" -eq 0 ] \
+               && cmd="$ssh_cmd ${user}@${system} $cmd"
+       echo -n "Start date (${what}-exec): "; date
+       echo "$cmd"
+       if [ "$DRYRUN" -eq 0 ]; then
+               $SHELL -c "$cmd"; local ret=$?
+       else
+               echo " *** Trial run, not executing ${what}-exec command!"
+               ret=0
+       fi
+       [ $ret -eq 0 ] \
+               && echo "The ${what}-exec command completed with status 0, OK." \
+               || echo "The ${what}-exec command completed with ERRORS, code $ret!"
+       return $ret
+}
+
+GetFS() {
+       local dir="$1"
+
+       while [ -n "$dir" ]; do
+               findmnt -fn -o FSTYPE --raw "$dir" 2>/dev/null; local r=$?
+               if [ $r -eq 0 ]; then
+                       return 0
+               elif [ $r -eq 127 ]; then
+                       echo "UNKNOWN"
+                       return 1
+               fi
+               dir=$(dirname "$dir") || return 1
+       done
+}
+
+CreateSubvolume() {
+       local volume="$1"
+       local fs
+       local dir
+
+       dir=$(dirname "$volume")
+       fs=$(GetFS "$dir")
+       case "$fs" in
+         "btrfs")
+               btrfs subvolume create "$volume"  >/dev/null || return 1
+               ;;
+         "zfs")
+               zfs create "$(echo "$volume" | cut -c2-)" || return 1
+               ;;
+         *)
+               echo "CreateSubvolume: Incompatible FS type \"$fs\" on \"$dir\"!"
+               return 9
+       esac
+       return 0
+}
+
+CloneSubvolume() {
+       local source="$1"
+       local volume="$2"
+       local snapshot="$3"
+       local dir
+       local fs
+       local link_name
+
+       dir=$(dirname "source")
+       fs=$(GetFS "$source")
+       case "$fs" in
+         "btrfs")
+               btrfs subvolume snapshot "$source" "$snapshot"  >/dev/null || return 1
+               ;;
+         "zfs")
+               zfs snapshot "$snapshot" || return 1
+               link_name="$(echo "$snapshot" | cut -d@ -f2-)"
+               ln -s \
+                       "current/.zfs/snapshot/$link_name" \
+                       "$(dirname "$volume")/$link_name"
+               ;;
+         *)
+               echo "CloneSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
+               return 9
+       esac
+       return 0
+}
+
+RenameSubvolume() {
+       local source="$1"
+       local target="$2"
+       local fs
+
+       fs=$(GetFS "$source")
+       case "$fs" in
+         "btrfs")
+               mv "$source" "$target" || return 1
+               ;;
+         "zfs")
+               zfs rename \
+                 "$(echo "$source" | cut -c2-)" \
+                 "$(echo "$target" | cut -c2-)" \
+                       || return 1
+               ;;
+         *)
+               echo "RenameSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
+               return 9
+       esac
+       return 0
+}
+
+DeleteSubvolume() {
+       local volume="$1"
+       local fs
+       local id
+       local snapshot
+
+       fs=$(GetFS "$volume")
+       case "$fs" in
+         "btrfs")
+               btrfs subvolume delete "$volume" >/dev/null || return 1
+               ;;
+         "zfs")
+               id="$(basename "$volume")"
+               if [ -h "$volume" ]; then
+                       snapshot="$(dirname "$volume")/current@$id"
+               else
+                       snapshot="$volume"
+               fi
+               zfs destroy -r "$(echo "$snapshot" | cut -c2-)" >/dev/null || return 1
+               [ -h "$volume" ] && rm "$volume"
+               ;;
+         *)
+               echo "DeleteSubvolume: Incompatible FS type \"$fs\" on \"$volume\"!"
+               return 9
+       esac
+       return 0
+}
+
+Initialize_Last_SysTarget_Snapshot() {
+       sys_target="$1"
+       unset last
+       unset snapshot
+
+       fs=$(GetFS "$sys_target")
+       case "$fs" in
+         "btrfs")
+               # Search directory of last generation, if any
+               # shellcheck disable=SC2012
+               last=$(ls -1d "$sys_target"/[0-9]* 2>/dev/null | sort -r | head -n1)
+               if [ -n "$last" ]; then
+                       if [ ! -d "$last" ]; then
+                               echo "Last snapshot \"$last\" seems not to be a directory!? \"$system\" skipped!"
+                               echo
+                               return 1
+                       fi
+               fi
+               sys_target="$sys_target/$(date +%Y%m%d-%H%M%S)"
+               snapshot="$sys_target"
+               ;;
+         "zfs")
+               # On ZFS, the last generation is always named "current"
+               if [ -e "$sys_target/current" ]; then
+                       last="$sys_target/current"
+                       if [ "$(uname)" = "Linux" ]; then
+                               date=$(LC_ALL=C stat "$1" | grep "^Modify: " \
+                                | cut -d':' -f2- | cut -d. -f1)
+                       else
+                               date=$(LC_ALL=C stat -f "%Sc" "$1")
+                       fi
+                       date=$(echo "$date" | sed -e's/^ //g' -e 's/[-:]//g' -e 's/ /-/g')
+
+               else
+                       last=""
+                       date="$(date +%Y%m%d-%H%M%S)"
+               fi
+               snapshot="$(echo "$sys_target/current" | cut -c2-)@$date"
+               sys_target="$sys_target/current"
+               ;;
+         *)
+               echo "Initialize_Last_SysTarget_Snapshot: Incompatible FS type \"$fs\" on \"$sys_target\"!"
+               return 1
+       esac
+       return 0
+}
+
+# Search configuration file (last one is used as default!)
+for conf in \
+       "/usr/local/etc/backup-script.conf" \
+       "/etc/backup-script.conf" \
+       "${conf_d}/backup-script.conf" \
+       "/usr/local/etc/backup-script.conf" \
+; do
+       [ -r "$conf" ] && break
+done
+
+# Read in configuration file
+config_info="Configuration file is \"$conf\""
+if [ -r "$conf" ]; then
+       # shellcheck source=/dev/null
+       source "$conf"
+else
+       config_info="${config_info} (not readable, using defaults)"
+fi
+config_info="${config_info},\nusing \"$conf_d\" as configuration directory."
+
 while [ $# -gt 0 ]; do
        case "$1" in
          "-n"|"--dry-run")
@@ -92,6 +301,10 @@ while [ $# -gt 0 ]; do
          "-p"|"--progress")
                VERBOSE=1; shift
                ;;
+         "-t"|"--tag")
+               shift; TAG="$1"; shift
+               [ -n "$TAG" ] || Usage
+               ;;
          "-"*)
                Usage
                ;;
@@ -100,36 +313,59 @@ while [ $# -gt 0 ]; do
        esac
 done
 
+echo -n "Started: "; date
+echo -e "$config_info"
+
+# Check rsync and its protocol version
+rsync=$(which "rsync" 2>/dev/null)
+if [ $? -ne 0 ]; then
+       echo "Failed to detect rsync(1)! Is it installed in your \$PATH?"
+       exit 1
+fi
+rsync_proto=$($rsync --version 2>/dev/null | head -n 1 | sed 's/.*  protocol version \([0-9]*\)$/\1/')
+if [ $? -ne 0 ]; then
+       echo "Failed to detect protocol version of $rsync!"
+       exit 1
+fi
+echo "Rsync command is $rsync, protocol version $rsync_proto."
+
+[[ -n "$TAG" ]] && echo "Running jobs tagged with \"$TAG\"."
+echo
+
 if [ $# -ge 1 ]; then
-       for s in $@; do
-               if [ ! -r "${CONF_D}/$s" ]; then
-                       echo "$NAME: Can' read \"${CONF_D}/$s\"!"
-                       exit 1
+       for s in "$@"; do
+               if [ ! -r "${conf_d}/$s" ]; then
+                       echo "$NAME: Can' read \"${conf_d}/$s\"!"
+                       exit 3
                fi
-               sys="$sys ${CONF_D}/$s"
+               sys+=("${conf_d}/$s")
        done
 else
-       sys=${CONF_D}/*
+       sys=("${conf_d}/"*)
 fi
 
-trap GotSignal SIGINT
-
-Log "Started ..."
-
-echo -n "Started: "; date
-echo
+if [ -n "$setup_exec" ]; then
+       echo "Executing \"$setup_exec\" ..."
+       sh -c $setup_exec
+       if [ $? -ne 0 ]; then
+               echo "Error: setup command failed!"; echo
+               echo "Aborting backup."; echo
+               exit 5
+       fi
+       sleep 2
+       echo
+fi
 
-[ -r "${CONF_D}/backup-script.conf" ] && source "${CONF_D}/backup-script.conf"
+trap GotSignal SIGINT
 
 # check and create PID file
 if [ -e "$PIDFILE" ]; then
-       Log "Lockfile \"$PIDFILE\" already exists. Aborting!"
        echo "Lockfile \"$PIDFILE\" already exists."
        echo "Is an other instance still running?"
        echo
        echo -n "Aborted: "; date
        echo
-       exit 3
+       exit 4
 fi
 touch "$PIDFILE" 2>/dev/null
 if [ $? -ne 0 ]; then
@@ -146,138 +382,407 @@ if [ -n "$pre_exec" ]; then
                echo "Error: pre-exec command failed!"; echo
                CleanUp
                echo "Aborting backup."; echo
-               exit 2
+               exit 5
        fi
        sleep 2
        echo
 fi
 
-for f in $sys; do
-       [ -r "$f" -a -f "$f" ] || continue
+for f in "${sys[@]}"; do
+       [[ -r "$f" && -f "$f" ]] || continue
 
-       system=`basename $f`
-       case "$system" in
+       fname=$(basename "$f")
+       case "$fname" in
                "backup-script.conf"|*.sh)
                        continue
                        ;;
        esac
 
        # Set global defaults
+       system="$fname"
+       backup_type="$default_backup_type"
        user="$default_user"
+       source_root="$default_source_root"
+       files="$default_files"
        target="$default_target"
        ssh_args_add="$default_ssh_args_add"
        rsync_args_add="$default_rsync_args_add"
        exclude_args_add="$default_exclude_args_add"
+       exclude_dirs_add="$default_exclude_dirs_add"
        compress="$default_compress"
        ping="$default_ping"
        local="$default_local"
+       generations="$default_generations"
+       job_pre_exec="$default_job_pre_exec"
+       job_post_exec="$default_job_post_exec"
+       tags="$default_tags"
+       io_timeout="$default_io_timeout"
+
+       # Compatibility with backup-pull(1) script: Save global values ...
+       pre_exec_saved="$pre_exec"
+       post_exec_saved="$post_exec"
+
+       # Compatibility with backup-pull(1) script: Set defaults
+       host=""
+       unset source
+       unset pre_exec
+       unset post_exec
 
        # Read in system configuration file
+       # shellcheck source=/dev/null
        source "$f"
 
+       # Compatibility with backup-pull(1) script: Fix up configuration
+       [[ "$system" = "$fname" && -n "$host" ]] \
+               && system="$host"
+       [[ "$source_root" = "$default_source_root" && -n "$source" ]] \
+               && source_root="$source"
+       [[ -z "$job_pre_exec" && -n "$pre_exec" ]] \
+               && job_pre_exec="$pre_exec"
+       [[ -z "$job_post_exec" && -n "$post_exec" ]] \
+               && job_post_exec="$post_exec"
+
+       # Compatibility with backup-pull(1) script: Restore global values ...
+       pre_exec="$pre_exec_saved"
+       post_exec="$post_exec_saved"
+
+       # Validate configuration
+       if [[ "$system" = "localhost" || "$system" = "127.0.0.1" ]]; then
+               # Local system
+               local=1
+               compress=0
+       fi
+
+       # Add "NONE" tag when no tags are given in the config file:
+       [[ -z "$tags" ]] && tags="NONE"
+       # Add "auto-tags":
+       [[ "$local" -eq 1 ]] && tags="$tags,LOCAL"
+       # Check tags
+       if [[ -n "$TAG" && "$TAG" != "ALL" ]]; then
+               echo "$tags" | grep -E "(^|,)$TAG(,|$)" >/dev/null 2>&1
+               if [ $? -ne 0 ]; then
+                       if [ "$DRYRUN" -ne 0 ]; then
+                               echo "Tags of system \"$system\" don't match \"$TAG\": \"$tags\". Skipped."
+                               echo
+                       fi
+                       continue
+               fi
+       fi
+
+       # Make sure "source_root" ends with a slash ("/")
+       case "$source_root" in
+         *"/")
+               ;;
+         *)
+               source_root="$source_root/"
+       esac
+
+       # Make sure "target" DOESN'T end with a slash ("/")
+       case "$target" in
+         "*/")
+               target=$( echo "$target" | sed -e 's/\/$//g' )
+               ;;
+       esac
+
+       [ "$system" = "$fname" ] \
+               && systxt="\"$system\"" \
+               || systxt="\"$fname\" [\"$system\"]"
        [ "$local" -eq 0 ] \
-               && MessageLog "Working on \"$system\" ..." \
-               || MessageLog "Working on \"$system\" (local system) ..."
+               && echo "Working on $systxt ..." \
+               || echo "Working on $systxt (local system) ..."
 
        count_all=$count_all+1
 
+       # Check if job is disabled
+       if [ "$backup_type" = "disabled" ]; then
+               echo "Job is DISABLED and will be skipped."
+               echo; continue
+       fi
+
+       count_enabled=$count_enabled+1
+
        # Check target directory
        if [ -z "$target" ]; then
-               MessageLog "No target directory specified for \"$system\"!? Skipped!"
+               echo "No target directory specified for \"$system\"!? Skipped!"
                echo; continue
        fi
        if [ ! -d "$target" ]; then
-               MessageLog "Target \"$target\" is not a directory!? \"$system\" skipped!"
+               echo "Target \"$target\" is not a directory!? \"$system\" skipped!"
                echo; continue
        fi
 
-       destdir="$target"
-       target="$target/$system"
-       [ "$DRYRUN" -gt 1 ] || mkdir -p "$target"
+       sys_target="$target/$fname"
+       sys_root="$sys_target"
+       if [[ "$DRYRUN" -eq 0 && ! -e "$sys_target" ]]; then
+               if [ $generations -gt 0 ]; then
+                       CreateSubvolume "$sys_target"
+               else
+                       mkdir -p "$sys_target"
+               fi
+               if [ $? -ne 0 ]; then
+                       echo "Can't create \"$sys_target\"!? \"$system\" skipped!"
+                       echo; continue
+               fi
+       fi
 
-       if [ "$local" -eq 0 -a "$ping" -ne 0 ]; then
+       if [[ "$local" -eq 0 && "$ping" -ne 0 ]]; then
                # Check if system is alive
                ping -c 1 "$system" >/dev/null 2>&1
                if [ $? -ne 0 ]; then
-                       MessageLog "Host \"$system\" seems not to be alive!? Skipped."
+                       echo "Host \"$system\" seems not to be alive!? Skipped."
                        echo; continue
                fi
-               Message "OK, host \"$system\" seems to be alive."
+               echo "OK, host \"$system\" seems to be alive."
+       fi
+
+       if [ $generations -gt 0 ]; then
+               # Make sure no old backup is stored in system directory
+               if [ -e "$sys_target/.stamp" ]; then
+                       # There seems to be a genearation-less backup in the
+                       # target directory!
+                       echo "Target directory \"$sys_target\" seems to be unclean!? \"$system\" skipped!"
+                       echo; continue
+               fi
+
+               Initialize_Last_SysTarget_Snapshot "$sys_target" || continue
+
+               if [[ -n "$last" && ! -e "$last/.stamp" ]]; then
+                       # Old backup directory without "stamp file", continue
+                       echo "Found incomplete snapshot in \"$last\", reusing and renaming it ..."
+                       RenameSubvolume "$last" "$sys_target"
+                       if [ $? -ne 0 ]; then
+                               echo "Failed to rename last snapshot \"$last\" to \"$sys_target\"!? \"$system\" skipped!"
+                               echo; continue
+                       fi
+               elif [ -n "$last" ]; then
+                       # Old backup directory found, create new snapshot
+                       echo "Found last snapshot in \"$last\"."
+                       if [ "$DRYRUN" -eq 0 ]; then
+                               CloneSubvolume "$last" "$sys_target" "$snapshot"; r=$?
+                               if [ $r -ne 0 ]; then
+                                       echo "Can't create snapshot \"$snapshot\" of \"$last\", code $r!? \"$system\" skipped!"
+                                       echo; continue
+                               fi
+                               echo "Created new snapshot in \"$snapshot\"."
+                       else
+                               echo " *** Trial run, not creating new snapshot in \"$snapshot\"!"
+                       fi
+               else
+                       # No old backup found, create new subvolume
+                       if [ "$DRYRUN" -eq 0 ]; then
+                               CreateSubvolume "$sys_target"; r=$?
+                               if [ $r -ne 0 ]; then
+                                       echo "Can't create subvolume \"$sys_target\", code $r!? \"$system\" skipped!"
+                                       echo; continue
+                               fi
+                               echo "Created new subvolume in \"$sys_target\"."
+                       else
+                               echo " *** Trial run, not creating new subvolume \"$sys_target\"!"
+                       fi
+               fi
        fi
 
        ssh_cmd="ssh"
        [ -n "$ssh_args_add" ] && ssh_cmd="$ssh_cmd $ssh_args_add"
 
-       cmd="rsync --archive"
-       [ "$compress" -ne 0 ] && cmd="$cmd --compress"
-       cmd="$cmd --rsh=\"$ssh_cmd\" --delete --delete-excluded --sparse"
-       [ "$VERBOSE" -gt 0 ] && cmd="$cmd --progress"
-       cmd="$cmd --exclude=/dev --exclude=/proc --exclude=/sys"
-       cmd="$cmd --exclude=/tmp --exclude=/var/tmp"
-       cmd="$cmd --exclude=/media --exclude=/mnt --exclude=/net"
-       cmd="$cmd --exclude=/var/cache/apt --exclude=/var/log"
-       [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add"
-       [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add"
+       # execute job "pre-exec" command, if any
+       if [ -n "$job_pre_exec" ]; then
+               ExecJob pre "$job_pre_exec" ; ret=$?
+               if [ $ret -ne 0 ]; then
+                       [ $ret -ne 99 ] && count_started=$count_started+1
+                       echo "Pre-exec command failed, \"$system\" skipped!"
+                       echo; continue
+               fi
+       fi
 
-       [ "$local" -eq 0 ] \
-               && cmd="$cmd ${user}@${system}:/ $target" \
-               || cmd="$cmd / $target"
+       # prepare (remote) command ...
+       if [[ "$backup_type" == "rsync" ]]; then
+               cmd="$rsync --archive --timeout=$io_timeout"
+               [ "$compress" -ne 0 ] && cmd="$cmd --compress"
+               [ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\""
+               cmd="$cmd --delete-during --delete-excluded --sparse"
+               if [ "$VERBOSE" -gt 0 ]; then
+                       [ "$rsync_proto" -ge 31 ] \
+                               && cmd="$cmd --info=progress2" \
+                               || cmd="$cmd --progress"
+               fi
+               set -f
+               if [ "$source_root" = "$default_source_root" ]; then
+                       for dir in \
+                               "/dev/**" \
+                               "/media/**" \
+                               "/mnt/**" \
+                               "/net/**" \
+                               "/proc/**" \
+                               "/run/**" \
+                               "/sys/**" \
+                               "/tmp/**" \
+                               "/var/cache/apt/**" \
+                               "/var/log/**" \
+                               "/var/tmp/**" \
+                       ; do
+                               cmd="$cmd --exclude=$dir"
+                       done
+               fi
+               [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add"
+               for dir in $exclude_dirs_add; do
+                       cmd="$cmd --exclude=$dir"
+               done
+               [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add"
+               set +f
+
+               [ "$local" -eq 0 ] \
+                       && cmd="$cmd ${user}@${system}:$source_root $sys_target/" \
+                       || cmd="$cmd $source_root $sys_target/"
+       elif [[ "$backup_type" == "scp" ]]; then
+               cmd="scp"
+               [ "$VERBOSE" -eq 0 ] && cmd="$cmd -q"
+               for file in $files; do
+                       cmd="$cmd ${user}@${system}:$file $sys_target/"
+               done
+       else
+               echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!"
+               echo; continue
+       fi
 
-       Message "Calling: $cmd"
+       echo "Backing up to \"$sys_target\" ..."
        echo -n "Start date: "; date
+       echo "$cmd"
        count_started=$count_started+1
-       rm -f "$target/.stamp"
-       
-       if [ "$DRYRUN" -lt 1 ]; then
+       ok=0
+
+       if [ "$DRYRUN" -eq 0 ]; then
+               stamp_file="$sys_target/.stamp"
+               rm -f "$stamp_file"
+
+               # Execute backup command:
+               start_t=$(date "+%s")
                $SHELL -c "$cmd"; ret=$?
-               echo "code=$ret" >"$target/.stamp"
+               end_t=$(date "+%s")
+
+               {
+                       echo "code=$ret"
+                       echo "start_t=$start_t"
+                       echo "end_t=$end_t"
+                       echo "cmd='$cmd'"
+                       echo "backup_host='$(hostname -f)'"
+                       echo "backup_user='$(id -un)'"
+               } >"$stamp_file"
        else
-               MessageLog " *** Trial run, not executing synchronization command!"
+               echo " *** Trial run, not executing save command!"
                ret=0
        fi
 
        if [ $ret -eq 20 ]; then
-               MessageLog "Backup of \"$system\" interrupted. Aborting ..."
-               CleanUp
-               exit 1
+               echo "Backup of \"$system\" interrupted. Aborting ..."
+               GotSignal
        fi
 
-       if [ $ret -eq 0 -o $ret -eq 24 ]; then
+       echo -n "End date: "; date
+       if [[ $ret -eq 0 || $ret -eq 24 ]]; then
                [ $ret -eq 24 ] && count_ok_vanished=$count_ok_vanished+1
 
-               MessageLog "System \"$system\" completed with status $ret, OK."
+               echo "System \"$system\" completed with status $ret, OK."
                [ "$DRYRUN" -gt 0 ] || count_ok=$count_ok+1
+               ok=1
        else
-               MessageLog "System \"$system\" completed with ERRORS, code $ret!"
+               echo "System \"$system\" completed with ERRORS, code $ret!"
+       fi
+
+       # execute job "post-exec" command, if any
+       if [ -n "$job_post_exec" ]; then
+               ExecJob post "$job_post_exec"
+       fi
+
+       if [ $generations -gt 0 ]; then
+               # Update "latest" symlink
+               if [ "$DRYRUN" -eq 0 ]; then
+                       rm -f "$sys_root/latest"
+                       ln -s "$sys_target" "$sys_root/latest"
+               fi
+               # Clean up old generations
+               declare -i gen_count=$generations+2
+               # shellcheck disable=SC2012
+               to_delete=$(ls -1t "$sys_root" 2>/dev/null | tail -n+$gen_count | sort)
+               if [[ -n "$to_delete" && $ok -eq 1 ]]; then
+                       [ "$DRYRUN" -eq 0 ] \
+                               && echo "Deleting old backup generations (keep $generations) ..." \
+                               || echo " *** Trial run, not deleting old generations:"
+                       for delete in $to_delete; do
+                               dir="$sys_root/$delete"
+                               if [ ! -e "$dir/.stamp" ]; then
+                                       echo "Not deleting \"$dir\", not a backup directory!?"
+                                       continue
+                               fi
+                               last=$(stat "$dir/.stamp" 2>/dev/null | grep "^Modify: " \
+                                | cut -d':' -f2- | cut -d. -f1)
+                               # shellcheck disable=SC2086
+                               echo "Removing backup from" $last "..."
+                               if [ "$DRYRUN" -eq 0 ]; then
+                                       DeleteSubvolume "$dir"
+                                       [ $? -eq 0 ] || \
+                                         echo "Failed to delete \"$dir\"!"
+                               fi
+                       done
+                       echo -n "Clean up finished: "; date
+               elif [ -n "$to_delete" ]; then
+                       echo "There have been errors, not cleaning up old generations!"
+               else
+                       echo "Nothing to clean up (keep up to $generations generations)."
+               fi
        fi
-       echo -n "End date: "; date
 
-       destinations="$destinations $destdir"
+       destinations="$destinations $target"
        echo
 done
 
 sync
 
-Log "Done: $count_all jobs, $count_started started, $count_ok completed without errors."
-
-paths=$( echo $destinations | sed -e 's/ /\n/g' | sort | uniq )
-if [ -n "$paths" ]; then
-       df -h $paths
-       echo
+if [ "$DRYRUN" -eq 0 ]; then
+       paths=""
+       paths_zfs=""
+       # shellcheck disable=SC2086
+       for dest in $(echo $destinations | sed -e 's/ /\n/g' | sort | uniq); do
+               fs=$(GetFS "$dest")
+               case $fs in
+                 "zfs" )
+                       paths_zfs="$paths_zfs $dest"
+                       ;;
+                 *)
+                       paths="$paths $dest"
+               esac
+       done
+       if [ -n "$paths" ]; then
+               # shellcheck disable=SC2086
+               df -h $paths
+               echo
+       fi
+       if [ -n "$paths_zfs" ]; then
+               # shellcheck disable=SC2086
+               zfs list $paths_zfs
+               echo
+       fi
 fi
 
 CleanUp
 
 echo -n "Done: "; date
 echo
-echo " - $count_all jobs defined,"
-echo " - $count_started jobs started,"
+[ $count_all -eq 1 ] && s="" || s="s"
+[ $count_enabled -eq $count_all ] \
+       && echo " - $count_all job$s defined (all enabled)," \
+       || echo " - $count_all job$s defined ($count_enabled enabled),"
+[ $count_started -eq 1 ] && s="" || s="s"
+echo " - $count_started job$s started,"
 echo " - $count_ok done without errors."
 echo
 
 if [ $count_started -ne $count_ok ]; then
        echo "----->  THERE HAVE BEEN ERRORS!  <-----"
        echo
+       exit 6
+elif [ $count_enabled -ne $count_started ]; then
+       exit 7
 fi
 
 # -eof-