bin/backup-script

   1 #!/bin/bash
   2 #
   3 # backup-script system for cloning systems using rsync
   4 # Copyright (c)2008-2016 Alexander Barton <alex@barton.de>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 # Please read the file COPYING, README and AUTHORS for more information.
  11 #
  12
  13 NAME=$(basename "$0")
  14 PIDFILE="/var/run/$NAME.pid"
  15
  16 DRYRUN=0
  17 VERBOSE=0
  18 TAG=""
  19
  20 export LC_ALL=C
  21
  22 declare -i count_all=0
  23 declare -i count_started=0
  24 declare -i count_ok=0
  25 declare -i count_ok_vanished=0
  26 declare -i count_enabled=0
  27
  28 destinations=""
  29
  30 # Default settings, can be overwritten in backup-script.conf:
  31 [ -d "/usr/local/etc/backup-script.d" ] \
  32         && conf_d="/usr/local/etc/backup-script.d" \
  33         || conf_d="/etc/backup-script.d"
  34 setup_exec=""
  35 pre_exec=""
  36 post_exec=""
  37 default_backup_type="rsync"
  38 default_source_root="/"
  39 default_files="running-config"
  40 default_target="/var/backups"
  41 default_user="root"
  42 default_ssh_args_add=""
  43 default_rsync_args_add=""
  44 default_exclude_args_add=""
  45 default_exclude_dirs_add=""
  46 default_compress=1
  47 default_ping=1
  48 default_local=0
  49 default_generations=0
  50 default_io_timeout="1800"
  51 default_job_pre_exec=""
  52 default_job_post_exec=""
  53 default_tags=""
  54
  55 Usage() {
  56         echo "Usage: $NAME [<options>] [<system> [<system> [...]]]"
  57         echo
  58         echo "  -n, --dry-run       Test run only, don't copy any data."
  59         echo "  -p, --progress      Show progress, see rsync(1)."
  60         echo "  -t TAG, --tag TAG   Only run jobs with tag TAG."
  61         echo
  62         echo "When no <system> is given, all defined systems are used."
  63         echo
  64         # shellcheck disable=SC2086
  65         echo -e $config_info
  66         echo
  67         exit 2
  68 }
  69
  70 CleanUp() {
  71         if [ -n "$post_exec" ]; then
  72                 echo "Executing \"$post_exec\" ..."
  73                 sh -c $post_exec
  74                 if [ $? -ne 0 ]; then
  75                         echo "Warning: post-exec command failed!"
  76                 fi
  77                 echo
  78         fi
  79         rm -f "$PIDFILE"
  80 }
  81
  82 GotSignal() {
  83         echo
  84         echo "--> Got break signal, cleaning up & aborting ..."
  85         echo
  86         CleanUp
  87         echo -n "Aborted: "; date
  88         echo
  89         sleep 3
  90         exit 9
  91 }
  92
  93 ExecJob() {
  94         local what="$1"
  95         local cmd="$2"
  96
  97         echo "Running job ${what}-exec command ..."
  98         [ "$local" -eq 0 ] \
  99                 && cmd="$ssh_cmd ${user}@${system} $cmd"
 100         echo -n "Start date (${what}-exec): "; date
 101         echo "$cmd"
 102         if [ "$DRYRUN" -eq 0 ]; then
 103                 $SHELL -c "$cmd"; local ret=$?
 104         else
 105                 echo " *** Trial run, not executing ${what}-exec command!"
 106                 ret=0
 107         fi
 108         [ $ret -eq 0 ] \
 109                 && echo "The ${what}-exec command completed with status 0, OK." \
 110                 || echo "The ${what}-exec command completed with ERRORS, code $ret!"
 111         return $ret
 112 }
 113
 114 GetFS() {
 115         local dir="$1"
 116
 117         while [ -n "$dir" ]; do
 118                 findmnt -fn -o FSTYPE --raw "$dir" 2>/dev/null; local r=$?
 119                 if [ $r -eq 0 ]; then
 120                         return 0
 121                 elif [ $r -eq 127 ]; then
 122                         echo "UNKNOWN"
 123                         return 1
 124                 fi
 125                 dir=$(dirname "$dir") || return 1
 126         done
 127 }
 128
 129 CreateSubvolume() {
 130         local volume="$1"
 131         local fs
 132         local dir
 133
 134         dir=$(dirname "$volume")
 135         fs=$(GetFS "$dir")
 136         case "$fs" in
 137           "btrfs")
 138                 btrfs subvolume create "$volume"  >/dev/null || return 1
 139                 ;;
 140           "zfs")
 141                 zfs create "$(echo "$volume" | cut -c2-)" || return 1
 142                 ;;
 143           *)
 144                 echo "CreateSubvolume: Incompatible FS type \"$fs\" on \"$dir\"!"
 145                 return 9
 146         esac
 147         return 0
 148 }
 149
 150 CloneSubvolume() {
 151         local source="$1"
 152         local volume="$2"
 153         local snapshot="$3"
 154         local dir
 155         local fs
 156         local link_name
 157
 158         dir=$(dirname "source")
 159         fs=$(GetFS "$source")
 160         case "$fs" in
 161           "btrfs")
 162                 btrfs subvolume snapshot "$source" "$snapshot"  >/dev/null || return 1
 163                 ;;
 164           "zfs")
 165                 zfs snapshot "$snapshot" || return 1
 166                 link_name="$(echo "$snapshot" | cut -d@ -f2-)"
 167                 ln -s \
 168                         "current/.zfs/snapshot/$link_name" \
 169                         "$(dirname "$volume")/$link_name"
 170                 ;;
 171           *)
 172                 echo "CloneSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
 173                 return 9
 174         esac
 175         return 0
 176 }
 177
 178 RenameSubvolume() {
 179         local source="$1"
 180         local target="$2"
 181         local fs
 182
 183         fs=$(GetFS "$source")
 184         case "$fs" in
 185           "btrfs")
 186                 mv "$source" "$target" || return 1
 187                 ;;
 188           "zfs")
 189                 zfs rename \
 190                   "$(echo "$source" | cut -c2-)" \
 191                   "$(echo "$target" | cut -c2-)" \
 192                         || return 1
 193                 ;;
 194           *)
 195                 echo "RenameSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
 196                 return 9
 197         esac
 198         return 0
 199 }
 200
 201 DeleteSubvolume() {
 202         local volume="$1"
 203         local fs
 204         local id
 205         local snapshot
 206
 207         fs=$(GetFS "$volume")
 208         case "$fs" in
 209           "btrfs")
 210                 btrfs subvolume delete "$volume" >/dev/null || return 1
 211                 ;;
 212           "zfs")
 213                 id="$(basename "$volume")"
 214                 if [ -h "$volume" ]; then
 215                         snapshot="$(dirname "$volume")/current@$id"
 216                 else
 217                         snapshot="$volume"
 218                 fi
 219                 zfs destroy -r "$(echo "$snapshot" | cut -c2-)" >/dev/null || return 1
 220                 [ -h "$volume" ] && rm "$volume"
 221                 ;;
 222           *)
 223                 echo "DeleteSubvolume: Incompatible FS type \"$fs\" on \"$volume\"!"
 224                 return 9
 225         esac
 226         return 0
 227 }
 228
 229 Initialize_Last_SysTarget_Snapshot() {
 230         sys_target="$1"
 231         unset last
 232         unset snapshot
 233
 234         fs=$(GetFS "$sys_target")
 235         case "$fs" in
 236           "btrfs")
 237                 # Search directory of last generation, if any
 238                 # shellcheck disable=SC2012
 239                 last=$(ls -1d "$sys_target"/[0-9]* 2>/dev/null | sort -r | head -n1)
 240                 if [ -n "$last" ]; then
 241                         if [ ! -d "$last" ]; then
 242                                 echo "Last snapshot \"$last\" seems not to be a directory!? \"$system\" skipped!"
 243                                 echo
 244                                 return 1
 245                         fi
 246                 fi
 247                 sys_target="$sys_target/$(date +%Y%m%d-%H%M%S)"
 248                 snapshot="$sys_target"
 249                 ;;
 250           "zfs")
 251                 # On ZFS, the last generation is always named "current"
 252                 if [ -e "$sys_target/current" ]; then
 253                         last="$sys_target/current"
 254                         if [ "$(uname)" = "Linux" ]; then
 255                                 date=$(LC_ALL=C stat "$1" | grep "^Modify: " \
 256                                  | cut -d':' -f2- | cut -d. -f1)
 257                         else
 258                                 date=$(LC_ALL=C stat -f "%Sc" "$1")
 259                         fi
 260                         date=$(echo "$date" | sed -e's/^ //g' -e 's/[-:]//g' -e 's/ /-/g')
 261
 262                 else
 263                         last=""
 264                         date="$(date +%Y%m%d-%H%M%S)"
 265                 fi
 266                 snapshot="$(echo "$sys_target/current" | cut -c2-)@$date"
 267                 sys_target="$sys_target/current"
 268                 ;;
 269           *)
 270                 echo "Initialize_Last_SysTarget_Snapshot: Incompatible FS type \"$fs\" on \"$sys_target\"!"
 271                 return 1
 272         esac
 273         return 0
 274 }
 275
 276 # Search configuration file (last one is used as default!)
 277 for conf in \
 278         "/usr/local/etc/backup-script.conf" \
 279         "/etc/backup-script.conf" \
 280         "${conf_d}/backup-script.conf" \
 281         "/usr/local/etc/backup-script.conf" \
 282 ; do
 283         [ -r "$conf" ] && break
 284 done
 285
 286 # Read in configuration file
 287 config_info="Configuration file is \"$conf\""
 288 if [ -r "$conf" ]; then
 289         # shellcheck source=/dev/null
 290         source "$conf"
 291 else
 292         config_info="${config_info} (not readable, using defaults)"
 293 fi
 294 config_info="${config_info},\nusing \"$conf_d\" as configuration directory."
 295
 296 while [ $# -gt 0 ]; do
 297         case "$1" in
 298           "-n"|"--dry-run")
 299                 DRYRUN=1; shift
 300                 ;;
 301           "-p"|"--progress")
 302                 VERBOSE=1; shift
 303                 ;;
 304           "-t"|"--tag")
 305                 shift; TAG="$1"; shift
 306                 [ -n "$TAG" ] || Usage
 307                 ;;
 308           "-"*)
 309                 Usage
 310                 ;;
 311           *)
 312                 break
 313         esac
 314 done
 315
 316 echo -n "Started: "; date
 317 echo -e "$config_info"
 318
 319 # Check rsync and its protocol version
 320 rsync=$(which "rsync" 2>/dev/null)
 321 if [ $? -ne 0 ]; then
 322         echo "Failed to detect rsync(1)! Is it installed in your \$PATH?"
 323         exit 1
 324 fi
 325 rsync_proto=$($rsync --version 2>/dev/null | head -n 1 | sed 's/.*  protocol version \([0-9]*\)$/\1/')
 326 if [ $? -ne 0 ]; then
 327         echo "Failed to detect protocol version of $rsync!"
 328         exit 1
 329 fi
 330 echo "Rsync command is $rsync, protocol version $rsync_proto."
 331
 332 [[ -n "$TAG" ]] && echo "Running jobs tagged with \"$TAG\"."
 333 echo
 334
 335 if [ $# -ge 1 ]; then
 336         for s in "$@"; do
 337                 if [ ! -r "${conf_d}/$s" ]; then
 338                         echo "$NAME: Can' read \"${conf_d}/$s\"!"
 339                         exit 3
 340                 fi
 341                 sys+=("${conf_d}/$s")
 342         done
 343 else
 344         sys=("${conf_d}/"*)
 345 fi
 346
 347 if [ -n "$setup_exec" ]; then
 348         echo "Executing \"$setup_exec\" ..."
 349         sh -c $setup_exec
 350         if [ $? -ne 0 ]; then
 351                 echo "Error: setup command failed!"; echo
 352                 echo "Aborting backup."; echo
 353                 exit 5
 354         fi
 355         sleep 2
 356         echo
 357 fi
 358
 359 trap GotSignal SIGINT
 360
 361 # check and create PID file
 362 if [ -e "$PIDFILE" ]; then
 363         echo "Lockfile \"$PIDFILE\" already exists."
 364         echo "Is an other instance still running?"
 365         echo
 366         echo -n "Aborted: "; date
 367         echo
 368         exit 4
 369 fi
 370 touch "$PIDFILE" 2>/dev/null
 371 if [ $? -ne 0 ]; then
 372         echo "Warning: can't create PID file \"$PIDFILE\"!"
 373         echo
 374 else
 375         echo "$$" >>"$PIDFILE"
 376 fi
 377
 378 if [ -n "$pre_exec" ]; then
 379         echo "Executing \"$pre_exec\" ..."
 380         sh -c $pre_exec
 381         if [ $? -ne 0 ]; then
 382                 echo "Error: pre-exec command failed!"; echo
 383                 CleanUp
 384                 echo "Aborting backup."; echo
 385                 exit 5
 386         fi
 387         sleep 2
 388         echo
 389 fi
 390
 391 for f in "${sys[@]}"; do
 392         [[ -r "$f" && -f "$f" ]] || continue
 393
 394         fname=$(basename "$f")
 395         case "$fname" in
 396                 "backup-script.conf"|*.sh)
 397                         continue
 398                         ;;
 399         esac
 400
 401         # Set global defaults
 402         system="$fname"
 403         backup_type="$default_backup_type"
 404         user="$default_user"
 405         source_root="$default_source_root"
 406         files="$default_files"
 407         target="$default_target"
 408         ssh_args_add="$default_ssh_args_add"
 409         rsync_args_add="$default_rsync_args_add"
 410         exclude_args_add="$default_exclude_args_add"
 411         exclude_dirs_add="$default_exclude_dirs_add"
 412         compress="$default_compress"
 413         ping="$default_ping"
 414         local="$default_local"
 415         generations="$default_generations"
 416         job_pre_exec="$default_job_pre_exec"
 417         job_post_exec="$default_job_post_exec"
 418         tags="$default_tags"
 419         io_timeout="$default_io_timeout"
 420
 421         # Compatibility with backup-pull(1) script: Save global values ...
 422         pre_exec_saved="$pre_exec"
 423         post_exec_saved="$post_exec"
 424
 425         # Compatibility with backup-pull(1) script: Set defaults
 426         host=""
 427         unset source
 428         unset pre_exec
 429         unset post_exec
 430
 431         # Read in system configuration file
 432         # shellcheck source=/dev/null
 433         source "$f"
 434
 435         # Compatibility with backup-pull(1) script: Fix up configuration
 436         [[ "$system" = "$fname" && -n "$host" ]] \
 437                 && system="$host"
 438         [[ "$source_root" = "$default_source_root" && -n "$source" ]] \
 439                 && source_root="$source"
 440         [[ -z "$job_pre_exec" && -n "$pre_exec" ]] \
 441                 && job_pre_exec="$pre_exec"
 442         [[ -z "$job_post_exec" && -n "$post_exec" ]] \
 443                 && job_post_exec="$post_exec"
 444
 445         # Compatibility with backup-pull(1) script: Restore global values ...
 446         pre_exec="$pre_exec_saved"
 447         post_exec="$post_exec_saved"
 448
 449         # Validate configuration
 450         if [[ "$system" = "localhost" || "$system" = "127.0.0.1" ]]; then
 451                 # Local system
 452                 local=1
 453                 compress=0
 454         fi
 455
 456         # Add "NONE" tag when no tags are given in the config file:
 457         [[ -z "$tags" ]] && tags="NONE"
 458         # Add "auto-tags":
 459         [[ "$local" -eq 1 ]] && tags="$tags,LOCAL"
 460         # Check tags
 461         if [[ -n "$TAG" && "$TAG" != "ALL" ]]; then
 462                 echo "$tags" | grep -E "(^|,)$TAG(,|$)" >/dev/null 2>&1
 463                 if [ $? -ne 0 ]; then
 464                         if [ "$DRYRUN" -ne 0 ]; then
 465                                 echo "Tags of system \"$system\" don't match \"$TAG\": \"$tags\". Skipped."
 466                                 echo
 467                         fi
 468                         continue
 469                 fi
 470         fi
 471
 472         # Make sure "source_root" ends with a slash ("/")
 473         case "$source_root" in
 474           *"/")
 475                 ;;
 476           *)
 477                 source_root="$source_root/"
 478         esac
 479
 480         # Make sure "target" DOESN'T end with a slash ("/")
 481         case "$target" in
 482           "*/")
 483                 target=$( echo "$target" | sed -e 's/\/$//g' )
 484                 ;;
 485         esac
 486
 487         [ "$system" = "$fname" ] \
 488                 && systxt="\"$system\"" \
 489                 || systxt="\"$fname\" [\"$system\"]"
 490         [ "$local" -eq 0 ] \
 491                 && echo "Working on $systxt ..." \
 492                 || echo "Working on $systxt (local system) ..."
 493
 494         count_all=$count_all+1
 495
 496         # Check if job is disabled
 497         if [ "$backup_type" = "disabled" ]; then
 498                 echo "Job is DISABLED and will be skipped."
 499                 echo; continue
 500         fi
 501
 502         count_enabled=$count_enabled+1
 503
 504         # Check target directory
 505         if [ -z "$target" ]; then
 506                 echo "No target directory specified for \"$system\"!? Skipped!"
 507                 echo; continue
 508         fi
 509         if [ ! -d "$target" ]; then
 510                 echo "Target \"$target\" is not a directory!? \"$system\" skipped!"
 511                 echo; continue
 512         fi
 513
 514         sys_target="$target/$fname"
 515         sys_root="$sys_target"
 516         if [[ "$DRYRUN" -eq 0 && ! -e "$sys_target" ]]; then
 517                 if [ $generations -gt 0 ]; then
 518                         CreateSubvolume "$sys_target"
 519                 else
 520                         mkdir -p "$sys_target"
 521                 fi
 522                 if [ $? -ne 0 ]; then
 523                         echo "Can't create \"$sys_target\"!? \"$system\" skipped!"
 524                         echo; continue
 525                 fi
 526         fi
 527
 528         if [[ "$local" -eq 0 && "$ping" -ne 0 ]]; then
 529                 # Check if system is alive
 530                 ping -c 1 "$system" >/dev/null 2>&1
 531                 if [ $? -ne 0 ]; then
 532                         echo "Host \"$system\" seems not to be alive!? Skipped."
 533                         echo; continue
 534                 fi
 535                 echo "OK, host \"$system\" seems to be alive."
 536         fi
 537
 538         if [ $generations -gt 0 ]; then
 539                 # Make sure no old backup is stored in system directory
 540                 if [ -e "$sys_target/.stamp" ]; then
 541                         # There seems to be a genearation-less backup in the
 542                         # target directory!
 543                         echo "Target directory \"$sys_target\" seems to be unclean!? \"$system\" skipped!"
 544                         echo; continue
 545                 fi
 546
 547                 Initialize_Last_SysTarget_Snapshot "$sys_target" || continue
 548
 549                 if [[ -n "$last" && ! -e "$last/.stamp" ]]; then
 550                         # Old backup directory without "stamp file", continue
 551                         echo "Found incomplete snapshot in \"$last\", reusing and renaming it ..."
 552                         RenameSubvolume "$last" "$sys_target"
 553                         if [ $? -ne 0 ]; then
 554                                 echo "Failed to rename last snapshot \"$last\" to \"$sys_target\"!? \"$system\" skipped!"
 555                                 echo; continue
 556                         fi
 557                 elif [ -n "$last" ]; then
 558                         # Old backup directory found, create new snapshot
 559                         echo "Found last snapshot in \"$last\"."
 560                         if [ "$DRYRUN" -eq 0 ]; then
 561                                 CloneSubvolume "$last" "$sys_target" "$snapshot"; r=$?
 562                                 if [ $r -ne 0 ]; then
 563                                         echo "Can't create snapshot \"$snapshot\" of \"$last\", code $r!? \"$system\" skipped!"
 564                                         echo; continue
 565                                 fi
 566                                 echo "Created new snapshot in \"$snapshot\"."
 567                         else
 568                                 echo " *** Trial run, not creating new snapshot in \"$snapshot\"!"
 569                         fi
 570                 else
 571                         # No old backup found, create new subvolume
 572                         if [ "$DRYRUN" -eq 0 ]; then
 573                                 CreateSubvolume "$sys_target"; r=$?
 574                                 if [ $r -ne 0 ]; then
 575                                         echo "Can't create subvolume \"$sys_target\", code $r!? \"$system\" skipped!"
 576                                         echo; continue
 577                                 fi
 578                                 echo "Created new subvolume in \"$sys_target\"."
 579                         else
 580                                 echo " *** Trial run, not creating new subvolume \"$sys_target\"!"
 581                         fi
 582                 fi
 583         fi
 584
 585         ssh_cmd="ssh"
 586         [ -n "$ssh_args_add" ] && ssh_cmd="$ssh_cmd $ssh_args_add"
 587
 588         # execute job "pre-exec" command, if any
 589         if [ -n "$job_pre_exec" ]; then
 590                 ExecJob pre "$job_pre_exec" ; ret=$?
 591                 if [ $ret -ne 0 ]; then
 592                         [ $ret -ne 99 ] && count_started=$count_started+1
 593                         echo "Pre-exec command failed, \"$system\" skipped!"
 594                         echo; continue
 595                 fi
 596         fi
 597
 598         # prepare (remote) command ...
 599         if [[ "$backup_type" == "rsync" ]]; then
 600                 cmd="$rsync --archive --timeout=$io_timeout"
 601                 [ "$compress" -ne 0 ] && cmd="$cmd --compress"
 602                 [ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\""
 603                 cmd="$cmd --delete-during --delete-excluded --sparse"
 604                 if [ "$VERBOSE" -gt 0 ]; then
 605                         [ "$rsync_proto" -ge 31 ] \
 606                                 && cmd="$cmd --info=progress2" \
 607                                 || cmd="$cmd --progress"
 608                 fi
 609                 set -f
 610                 if [ "$source_root" = "$default_source_root" ]; then
 611                         for dir in \
 612                                 "/dev/**" \
 613                                 "/media/**" \
 614                                 "/mnt/**" \
 615                                 "/net/**" \
 616                                 "/proc/**" \
 617                                 "/run/**" \
 618                                 "/sys/**" \
 619                                 "/tmp/**" \
 620                                 "/var/cache/apt/**" \
 621                                 "/var/log/**" \
 622                                 "/var/tmp/**" \
 623                         ; do
 624                                 cmd="$cmd --exclude=$dir"
 625                         done
 626                 fi
 627                 [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add"
 628                 for dir in $exclude_dirs_add; do
 629                         cmd="$cmd --exclude=$dir"
 630                 done
 631                 [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add"
 632                 set +f
 633
 634                 [ "$local" -eq 0 ] \
 635                         && cmd="$cmd ${user}@${system}:$source_root $sys_target/" \
 636                         || cmd="$cmd $source_root $sys_target/"
 637         elif [[ "$backup_type" == "scp" ]]; then
 638                 cmd="scp"
 639                 [ "$VERBOSE" -eq 0 ] && cmd="$cmd -q"
 640                 for file in $files; do
 641                         cmd="$cmd ${user}@${system}:$file $sys_target/"
 642                 done
 643         else
 644                 echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!"
 645                 echo; continue
 646         fi
 647
 648         echo "Backing up to \"$sys_target\" ..."
 649         echo -n "Start date: "; date
 650         echo "$cmd"
 651         count_started=$count_started+1
 652         ok=0
 653
 654         if [ "$DRYRUN" -eq 0 ]; then
 655                 stamp_file="$sys_target/.stamp"
 656                 rm -f "$stamp_file"
 657
 658                 # Execute backup command:
 659                 start_t=$(date "+%s")
 660                 $SHELL -c "$cmd"; ret=$?
 661                 end_t=$(date "+%s")
 662
 663                 {
 664                         echo "code=$ret"
 665                         echo "start_t=$start_t"
 666                         echo "end_t=$end_t"
 667                         echo "cmd='$cmd'"
 668                         echo "backup_host='$(hostname -f)'"
 669                         echo "backup_user='$(id -un)'"
 670                 } >"$stamp_file"
 671         else
 672                 echo " *** Trial run, not executing save command!"
 673                 ret=0
 674         fi
 675
 676         if [ $ret -eq 20 ]; then
 677                 echo "Backup of \"$system\" interrupted. Aborting ..."
 678                 GotSignal
 679         fi
 680
 681         echo -n "End date: "; date
 682         if [[ $ret -eq 0 || $ret -eq 24 ]]; then
 683                 [ $ret -eq 24 ] && count_ok_vanished=$count_ok_vanished+1
 684
 685                 echo "System \"$system\" completed with status $ret, OK."
 686                 [ "$DRYRUN" -gt 0 ] || count_ok=$count_ok+1
 687                 ok=1
 688         else
 689                 echo "System \"$system\" completed with ERRORS, code $ret!"
 690         fi
 691
 692         # execute job "post-exec" command, if any
 693         if [ -n "$job_post_exec" ]; then
 694                 ExecJob post "$job_post_exec"
 695         fi
 696
 697         if [ $generations -gt 0 ]; then
 698                 # Update "latest" symlink
 699                 if [ "$DRYRUN" -eq 0 ]; then
 700                         rm -f "$sys_root/latest"
 701                         ln -s "$sys_target" "$sys_root/latest"
 702                 fi
 703                 # Clean up old generations
 704                 declare -i gen_count=$generations+2
 705                 # shellcheck disable=SC2012
 706                 to_delete=$(ls -1t "$sys_root" 2>/dev/null | tail -n+$gen_count | sort)
 707                 if [[ -n "$to_delete" && $ok -eq 1 ]]; then
 708                         [ "$DRYRUN" -eq 0 ] \
 709                                 && echo "Deleting old backup generations (keep $generations) ..." \
 710                                 || echo " *** Trial run, not deleting old generations:"
 711                         for delete in $to_delete; do
 712                                 dir="$sys_root/$delete"
 713                                 if [ ! -e "$dir/.stamp" ]; then
 714                                         echo "Not deleting \"$dir\", not a backup directory!?"
 715                                         continue
 716                                 fi
 717                                 last=$(stat "$dir/.stamp" 2>/dev/null | grep "^Modify: " \
 718                                  | cut -d':' -f2- | cut -d. -f1)
 719                                 # shellcheck disable=SC2086
 720                                 echo "Removing backup from" $last "..."
 721                                 if [ "$DRYRUN" -eq 0 ]; then
 722                                         DeleteSubvolume "$dir"
 723                                         [ $? -eq 0 ] || \
 724                                           echo "Failed to delete \"$dir\"!"
 725                                 fi
 726                         done
 727                         echo -n "Clean up finished: "; date
 728                 elif [ -n "$to_delete" ]; then
 729                         echo "There have been errors, not cleaning up old generations!"
 730                 else
 731                         echo "Nothing to clean up (keep up to $generations generations)."
 732                 fi
 733         fi
 734
 735         destinations="$destinations $target"
 736         echo
 737 done
 738
 739 sync
 740
 741 if [ "$DRYRUN" -eq 0 ]; then
 742         paths=""
 743         paths_zfs=""
 744         # shellcheck disable=SC2086
 745         for dest in $(echo $destinations | sed -e 's/ /\n/g' | sort | uniq); do
 746                 fs=$(GetFS "$dest")
 747                 case $fs in
 748                   "zfs" )
 749                         paths_zfs="$paths_zfs $dest"
 750                         ;;
 751                   *)
 752                         paths="$paths $dest"
 753                 esac
 754         done
 755         if [ -n "$paths" ]; then
 756                 # shellcheck disable=SC2086
 757                 df -h $paths
 758                 echo
 759         fi
 760         if [ -n "$paths_zfs" ]; then
 761                 # shellcheck disable=SC2086
 762                 zfs list $paths_zfs
 763                 echo
 764         fi
 765 fi
 766
 767 CleanUp
 768
 769 echo -n "Done: "; date
 770 echo
 771 [ $count_all -eq 1 ] && s="" || s="s"
 772 [ $count_enabled -eq $count_all ] \
 773         && echo " - $count_all job$s defined (all enabled)," \
 774         || echo " - $count_all job$s defined ($count_enabled enabled),"
 775 [ $count_started -eq 1 ] && s="" || s="s"
 776 echo " - $count_started job$s started,"
 777 echo " - $count_ok done without errors."
 778 echo
 779
 780 if [ $count_started -ne $count_ok ]; then
 781         echo "----->  THERE HAVE BEEN ERRORS!  <-----"
 782         echo
 783         exit 6
 784 elif [ $count_enabled -ne $count_started ]; then
 785         exit 7
 786 fi
 787
 788 # -eof-