bin/backup-script

   1 #!/bin/bash
   2 #
   3 # backup-script system for cloning systems using rsync
   4 # Copyright (c)2008-2016 Alexander Barton <alex@barton.de>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 # Please read the file COPYING, README and AUTHORS for more information.
  11 #
  12
  13 NAME=$(basename "$0")
  14 PIDFILE="/var/run/$NAME.pid"
  15
  16 DRYRUN=0
  17 VERBOSE=0
  18 TAG=""
  19 PREPOSTEXEC=1
  20
  21 export LC_ALL=C
  22
  23 declare -i count_all=0
  24 declare -i count_started=0
  25 declare -i count_ok=0
  26 declare -i count_ok_vanished=0
  27 declare -i count_enabled=0
  28
  29 destinations=""
  30
  31 # Default settings, can be overwritten in backup-script.conf:
  32 [ -d "/usr/local/etc/backup-script.d" ] \
  33         && conf_d="/usr/local/etc/backup-script.d" \
  34         || conf_d="/etc/backup-script.d"
  35 setup_exec=""
  36 pre_exec=""
  37 post_exec=""
  38 default_backup_type="rsync"
  39 default_source_root="/"
  40 default_files="running-config"
  41 default_target="/var/backups"
  42 default_user="root"
  43 default_ssh_args_add=""
  44 default_rsync_args_add=""
  45 default_exclude_args_add=""
  46 default_exclude_dirs_add=""
  47 default_compress=1
  48 default_ping=1
  49 default_local=0
  50 default_generations=0
  51 default_io_timeout="1800"
  52 default_job_pre_exec=""
  53 default_job_post_exec=""
  54 default_tags=""
  55
  56 Usage() {
  57         echo "Usage: $NAME [<options>] [<job> [<job> [...]]]"
  58         echo
  59         echo "  -n, --dry-run       Test run only, don't copy any data."
  60         echo "  -p, --progress      Show progress, see rsync(1)."
  61         echo "  -t TAG, --tag TAG   Only run jobs with tag TAG."
  62         echo "  -x, --no-exec       Don't run global pre-/post-exec commands."
  63         echo
  64         echo "When no <job> is given, all defined systems are used."
  65         echo
  66         # shellcheck disable=SC2086
  67         echo -e $config_info
  68         echo
  69         exit 2
  70 }
  71
  72 CleanUp() {
  73         if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then
  74                 echo "Executing \"$post_exec\" ..."
  75                 sh -c $post_exec
  76                 if [ $? -ne 0 ]; then
  77                         echo "Warning: post-exec command failed!"
  78                 fi
  79                 echo
  80         fi
  81         rm -f "$PIDFILE"
  82 }
  83
  84 GotSignal() {
  85         echo
  86         echo "--> Got signal, cleaning up & aborting ..."
  87         echo
  88         CleanUp
  89         echo -n "Aborted: "; date
  90         echo
  91         sleep 3
  92         exit 9
  93 }
  94
  95 ExecJob() {
  96         local what="$1"
  97         local cmd="$2"
  98
  99         echo "Running job ${what}-exec command ..."
 100         [ "$local" -eq 0 ] \
 101                 && cmd="$ssh_cmd ${user}@${system} $cmd"
 102         echo -n "Start date (${what}-exec): "; date
 103         echo "$cmd"
 104         if [ "$DRYRUN" -eq 0 ]; then
 105                 $SHELL -c "$cmd"; local ret=$?
 106         else
 107                 echo " *** Trial run, not executing ${what}-exec command!"
 108                 ret=0
 109         fi
 110         [ $ret -eq 0 ] \
 111                 && echo "The ${what}-exec command completed with status 0, OK." \
 112                 || echo "The ${what}-exec command completed with ERRORS, code $ret!"
 113         return $ret
 114 }
 115
 116 GetFS() {
 117         local dir="$1"
 118
 119         while [ -n "$dir" ]; do
 120                 findmnt -fn -o FSTYPE --raw "$dir" 2>/dev/null; local r=$?
 121                 if [ $r -eq 0 ]; then
 122                         return 0
 123                 elif [ $r -eq 127 ]; then
 124                         echo "UNKNOWN"
 125                         return 1
 126                 fi
 127                 dir=$(dirname "$dir") || return 1
 128         done
 129 }
 130
 131 CreateSubvolume() {
 132         local volume="$1"
 133         local fs
 134         local dir
 135
 136         dir=$(dirname "$volume")
 137         fs=$(GetFS "$dir")
 138         case "$fs" in
 139           "btrfs")
 140                 btrfs subvolume create "$volume"  >/dev/null || return 1
 141                 ;;
 142           "zfs")
 143                 zfs create "$(echo "$volume" | cut -c2-)" || return 1
 144                 ;;
 145           *)
 146                 echo "CreateSubvolume: Incompatible FS type \"$fs\" on \"$dir\"!"
 147                 return 9
 148         esac
 149         return 0
 150 }
 151
 152 CloneSubvolume() {
 153         local source="$1"
 154         local volume="$2"
 155         local snapshot="$3"
 156         local dir
 157         local fs
 158         local link_name
 159
 160         dir=$(dirname "source")
 161         fs=$(GetFS "$source")
 162         case "$fs" in
 163           "btrfs")
 164                 btrfs subvolume snapshot "$source" "$snapshot"  >/dev/null || return 1
 165                 ;;
 166           "zfs")
 167                 zfs snapshot "$snapshot" || return 1
 168                 link_name="$(echo "$snapshot" | cut -d@ -f2-)"
 169                 ln -s \
 170                         "current/.zfs/snapshot/$link_name" \
 171                         "$(dirname "$volume")/$link_name"
 172                 ;;
 173           *)
 174                 echo "CloneSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
 175                 return 9
 176         esac
 177         return 0
 178 }
 179
 180 RenameSubvolume() {
 181         local source="$1"
 182         local target="$2"
 183         local fs
 184
 185         fs=$(GetFS "$source")
 186         case "$fs" in
 187           "btrfs")
 188                 mv "$source" "$target" || return 1
 189                 ;;
 190           "zfs")
 191                 zfs rename \
 192                   "$(echo "$source" | cut -c2-)" \
 193                   "$(echo "$target" | cut -c2-)" \
 194                         || return 1
 195                 ;;
 196           *)
 197                 echo "RenameSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
 198                 return 9
 199         esac
 200         return 0
 201 }
 202
 203 DeleteSubvolume() {
 204         local volume="$1"
 205         local fs
 206         local id
 207         local snapshot
 208
 209         fs=$(GetFS "$volume")
 210         case "$fs" in
 211           "btrfs")
 212                 btrfs subvolume delete "$volume" >/dev/null || return 1
 213                 ;;
 214           "zfs")
 215                 id="$(basename "$volume")"
 216                 if [ -h "$volume" ]; then
 217                         snapshot="$(dirname "$volume")/current@$id"
 218                 else
 219                         snapshot="$volume"
 220                 fi
 221                 zfs destroy -r "$(echo "$snapshot" | cut -c2-)" >/dev/null || return 1
 222                 [ -h "$volume" ] && rm "$volume"
 223                 ;;
 224           *)
 225                 echo "DeleteSubvolume: Incompatible FS type \"$fs\" on \"$volume\"!"
 226                 return 9
 227         esac
 228         return 0
 229 }
 230
 231 Initialize_Last_SysTarget_Snapshot() {
 232         sys_target="$1"
 233         unset last
 234         unset snapshot
 235
 236         fs=$(GetFS "$sys_target")
 237         case "$fs" in
 238           "btrfs")
 239                 # Search directory of last generation, if any
 240                 # shellcheck disable=SC2012
 241                 last=$(ls -1d "$sys_target"/[0-9]* 2>/dev/null | sort -r | head -n1)
 242                 if [ -n "$last" ]; then
 243                         if [ ! -d "$last" ]; then
 244                                 echo "Last snapshot \"$last\" seems not to be a directory!? \"$system\" skipped!"
 245                                 echo
 246                                 return 1
 247                         fi
 248                 fi
 249                 sys_target="$sys_target/$(date +%Y%m%d-%H%M%S)"
 250                 snapshot="$sys_target"
 251                 ;;
 252           "zfs")
 253                 # On ZFS, the last generation is always named "current"
 254                 if [ -e "$sys_target/current" ]; then
 255                         last="$sys_target/current"
 256                         if [ "$(uname)" = "Linux" ]; then
 257                                 date=$(LC_ALL=C stat "$1" | grep "^Modify: " \
 258                                  | cut -d':' -f2- | cut -d. -f1)
 259                         else
 260                                 date=$(LC_ALL=C stat -f "%Sc" "$1")
 261                         fi
 262                         date=$(echo "$date" | sed -e's/^ //g' -e 's/[-:]//g' -e 's/ /-/g')
 263
 264                 else
 265                         last=""
 266                         date="$(date +%Y%m%d-%H%M%S)"
 267                 fi
 268                 snapshot="$(echo "$sys_target/current" | cut -c2-)@$date"
 269                 sys_target="$sys_target/current"
 270                 ;;
 271           *)
 272                 echo "Initialize_Last_SysTarget_Snapshot: Incompatible FS type \"$fs\" on \"$sys_target\"!"
 273                 return 1
 274         esac
 275         return 0
 276 }
 277
 278 # Search configuration file (last one is used as default!)
 279 for conf in \
 280         "/usr/local/etc/backup-script.conf" \
 281         "/etc/backup-script.conf" \
 282         "${conf_d}/backup-script.conf" \
 283         "/usr/local/etc/backup-script.conf" \
 284 ; do
 285         [ -r "$conf" ] && break
 286 done
 287
 288 # Read in configuration file
 289 config_info="Configuration file is \"$conf\""
 290 if [ -r "$conf" ]; then
 291         # shellcheck source=/dev/null
 292         source "$conf"
 293 else
 294         config_info="${config_info} (not readable, using defaults)"
 295 fi
 296 config_info="${config_info},\nusing \"$conf_d\" as configuration directory."
 297
 298 while [ $# -gt 0 ]; do
 299         case "$1" in
 300           "-n"|"--dry-run")
 301                 DRYRUN=1; shift
 302                 ;;
 303           "-p"|"--progress")
 304                 VERBOSE=1; shift
 305                 ;;
 306           "-t"|"--tag")
 307                 shift; TAG="$1"; shift
 308                 [ -n "$TAG" ] || Usage
 309                 ;;
 310           "-x"|"--no-exec")
 311                 PREPOSTEXEC=0; shift
 312                 ;;
 313           "-"*)
 314                 Usage
 315                 ;;
 316           *)
 317                 break
 318         esac
 319 done
 320
 321 echo -n "Started: "; date
 322 echo -e "$config_info"
 323
 324 # Check rsync and its protocol version
 325 rsync=$(which "rsync" 2>/dev/null)
 326 if [ $? -ne 0 ]; then
 327         echo "Failed to detect rsync(1)! Is it installed in your \$PATH?"
 328         exit 1
 329 fi
 330 rsync_proto=$($rsync --version 2>/dev/null | head -n 1 | sed 's/.*  protocol version \([0-9]*\)$/\1/')
 331 if [ $? -ne 0 ]; then
 332         echo "Failed to detect protocol version of $rsync!"
 333         exit 1
 334 fi
 335 echo "Rsync command is $rsync, protocol version $rsync_proto."
 336
 337 [[ -n "$TAG" ]] && echo "Running jobs tagged with \"$TAG\"."
 338 echo
 339
 340 if [ $# -ge 1 ]; then
 341         for s in "$@"; do
 342                 if [ ! -r "${conf_d}/$s" ]; then
 343                         echo "$NAME: Can' read \"${conf_d}/$s\"!"
 344                         exit 3
 345                 fi
 346                 sys+=("${conf_d}/$s")
 347         done
 348 else
 349         sys=("${conf_d}/"*)
 350 fi
 351
 352 if [[ -n "$setup_exec" && $PREPOSTEXEC -ne 0 ]]; then
 353         echo "Executing \"$setup_exec\" ..."
 354         sh -c $setup_exec
 355         if [ $? -ne 0 ]; then
 356                 echo "Error: setup command failed!"; echo
 357                 echo "Aborting backup."; echo
 358                 exit 5
 359         fi
 360         sleep 2
 361         echo
 362 fi
 363
 364 trap GotSignal SIGINT SIGTERM
 365
 366 # check and create PID file
 367 if [ -e "$PIDFILE" ]; then
 368         echo "Lockfile \"$PIDFILE\" already exists."
 369         echo "Is an other instance still running?"
 370         echo
 371         echo -n "Aborted: "; date
 372         echo
 373         exit 4
 374 fi
 375 touch "$PIDFILE" 2>/dev/null
 376 if [ $? -ne 0 ]; then
 377         echo "Warning: can't create PID file \"$PIDFILE\"!"
 378         echo
 379 else
 380         echo "$$" >>"$PIDFILE"
 381 fi
 382
 383 if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then
 384         echo "Executing \"$pre_exec\" ..."
 385         sh -c $pre_exec
 386         if [ $? -ne 0 ]; then
 387                 echo "Error: pre-exec command failed!"; echo
 388                 CleanUp
 389                 echo "Aborting backup."; echo
 390                 exit 5
 391         fi
 392         sleep 2
 393         echo
 394 fi
 395
 396 for f in "${sys[@]}"; do
 397         [[ -r "$f" && -f "$f" ]] || continue
 398
 399         fname=$(basename "$f")
 400         case "$fname" in
 401                 "backup-script.conf"|*.sh)
 402                         continue
 403                         ;;
 404         esac
 405
 406         # Set global defaults
 407         system="$fname"
 408         backup_type="$default_backup_type"
 409         user="$default_user"
 410         source_root="$default_source_root"
 411         files="$default_files"
 412         target="$default_target"
 413         ssh_args_add="$default_ssh_args_add"
 414         rsync_args_add="$default_rsync_args_add"
 415         exclude_args_add="$default_exclude_args_add"
 416         exclude_dirs_add="$default_exclude_dirs_add"
 417         compress="$default_compress"
 418         ping="$default_ping"
 419         local="$default_local"
 420         generations="$default_generations"
 421         job_pre_exec="$default_job_pre_exec"
 422         job_post_exec="$default_job_post_exec"
 423         tags="$default_tags"
 424         io_timeout="$default_io_timeout"
 425
 426         # Compatibility with backup-pull(1) script: Save global values ...
 427         pre_exec_saved="$pre_exec"
 428         post_exec_saved="$post_exec"
 429
 430         # Compatibility with backup-pull(1) script: Set defaults
 431         host=""
 432         unset source
 433         unset pre_exec
 434         unset post_exec
 435
 436         # Read in system configuration file
 437         # shellcheck source=/dev/null
 438         source "$f"
 439
 440         # Compatibility with backup-pull(1) script: Fix up configuration
 441         [[ "$system" = "$fname" && -n "$host" ]] \
 442                 && system="$host"
 443         [[ "$source_root" = "$default_source_root" && -n "$source" ]] \
 444                 && source_root="$source"
 445         [[ -z "$job_pre_exec" && -n "$pre_exec" ]] \
 446                 && job_pre_exec="$pre_exec"
 447         [[ -z "$job_post_exec" && -n "$post_exec" ]] \
 448                 && job_post_exec="$post_exec"
 449
 450         # Compatibility with backup-pull(1) script: Restore global values ...
 451         pre_exec="$pre_exec_saved"
 452         post_exec="$post_exec_saved"
 453
 454         # Validate configuration
 455         if [[ "$system" = "localhost" || "$system" = "127.0.0.1" ]]; then
 456                 # Local system
 457                 local=1
 458                 compress=0
 459         fi
 460
 461         # Add "NONE" tag when no tags are given in the config file:
 462         [[ -z "$tags" ]] && tags="NONE"
 463         # Add "auto-tags":
 464         [[ "$local" -eq 1 ]] && tags="$tags,LOCAL"
 465         # Check tags
 466         if [[ -n "$TAG" && "$TAG" != "ALL" ]]; then
 467                 echo "$tags" | grep -E "(^|,)$TAG(,|$)" >/dev/null 2>&1
 468                 if [ $? -ne 0 ]; then
 469                         if [ "$DRYRUN" -ne 0 ]; then
 470                                 echo "Tags of system \"$system\" don't match \"$TAG\": \"$tags\". Skipped."
 471                                 echo
 472                         fi
 473                         continue
 474                 fi
 475         fi
 476
 477         # Make sure "source_root" ends with a slash ("/")
 478         case "$source_root" in
 479           *"/")
 480                 ;;
 481           *)
 482                 source_root="$source_root/"
 483         esac
 484
 485         # Make sure "target" DOESN'T end with a slash ("/")
 486         case "$target" in
 487           "*/")
 488                 target=$( echo "$target" | sed -e 's/\/$//g' )
 489                 ;;
 490         esac
 491
 492         [ "$system" = "$fname" ] \
 493                 && systxt="\"$system\"" \
 494                 || systxt="\"$fname\" [\"$system\"]"
 495         [ "$local" -eq 0 ] \
 496                 && echo "Working on $systxt ..." \
 497                 || echo "Working on $systxt (local system) ..."
 498
 499         count_all=$count_all+1
 500
 501         # Check if job is disabled
 502         if [ "$backup_type" = "disabled" ]; then
 503                 echo "Job is DISABLED and will be skipped."
 504                 echo; continue
 505         fi
 506
 507         count_enabled=$count_enabled+1
 508
 509         # Check target directory
 510         if [ -z "$target" ]; then
 511                 echo "No target directory specified for \"$system\"!? Skipped!"
 512                 echo; continue
 513         fi
 514         if [ ! -d "$target" ]; then
 515                 echo "Target \"$target\" is not a directory!? \"$system\" skipped!"
 516                 echo; continue
 517         fi
 518
 519         sys_target="$target/$fname"
 520         sys_root="$sys_target"
 521         if [[ "$DRYRUN" -eq 0 && ! -e "$sys_target" ]]; then
 522                 if [ $generations -gt 0 ]; then
 523                         CreateSubvolume "$sys_target"
 524                 else
 525                         mkdir -p "$sys_target"
 526                 fi
 527                 if [ $? -ne 0 ]; then
 528                         echo "Can't create \"$sys_target\"!? \"$system\" skipped!"
 529                         echo; continue
 530                 fi
 531         fi
 532
 533         if [[ "$local" -eq 0 && "$ping" -ne 0 ]]; then
 534                 # Check if system is alive
 535                 ping -c 1 "$system" >/dev/null 2>&1
 536                 if [ $? -ne 0 ]; then
 537                         echo "Host \"$system\" seems not to be alive!? Skipped."
 538                         echo; continue
 539                 fi
 540                 echo "OK, host \"$system\" seems to be alive."
 541         fi
 542
 543         if [ $generations -gt 0 ]; then
 544                 # Make sure no old backup is stored in system directory
 545                 if [ -e "$sys_target/.stamp" ]; then
 546                         # There seems to be a genearation-less backup in the
 547                         # target directory!
 548                         echo "Target directory \"$sys_target\" seems to be unclean!? \"$system\" skipped!"
 549                         echo; continue
 550                 fi
 551
 552                 Initialize_Last_SysTarget_Snapshot "$sys_target" || continue
 553
 554                 if [[ -n "$last" && ! -e "$last/.stamp" ]]; then
 555                         # Old backup directory without "stamp file", continue
 556                         echo "Found incomplete snapshot in \"$last\", reusing and renaming it ..."
 557                         RenameSubvolume "$last" "$sys_target"
 558                         if [ $? -ne 0 ]; then
 559                                 echo "Failed to rename last snapshot \"$last\" to \"$sys_target\"!? \"$system\" skipped!"
 560                                 echo; continue
 561                         fi
 562                 elif [ -n "$last" ]; then
 563                         # Old backup directory found, create new snapshot
 564                         echo "Found last snapshot in \"$last\"."
 565                         if [ "$DRYRUN" -eq 0 ]; then
 566                                 CloneSubvolume "$last" "$sys_target" "$snapshot"; r=$?
 567                                 if [ $r -ne 0 ]; then
 568                                         echo "Can't create snapshot \"$snapshot\" of \"$last\", code $r!? \"$system\" skipped!"
 569                                         echo; continue
 570                                 fi
 571                                 echo "Created new snapshot in \"$snapshot\"."
 572                         else
 573                                 echo " *** Trial run, not creating new snapshot in \"$snapshot\"!"
 574                         fi
 575                 else
 576                         # No old backup found, create new subvolume
 577                         if [ "$DRYRUN" -eq 0 ]; then
 578                                 CreateSubvolume "$sys_target"; r=$?
 579                                 if [ $r -ne 0 ]; then
 580                                         echo "Can't create subvolume \"$sys_target\", code $r!? \"$system\" skipped!"
 581                                         echo; continue
 582                                 fi
 583                                 echo "Created new subvolume in \"$sys_target\"."
 584                         else
 585                                 echo " *** Trial run, not creating new subvolume \"$sys_target\"!"
 586                         fi
 587                 fi
 588         fi
 589
 590         ssh_cmd="ssh"
 591         [ -n "$ssh_args_add" ] && ssh_cmd="$ssh_cmd $ssh_args_add"
 592
 593         # execute job "pre-exec" command, if any
 594         if [ -n "$job_pre_exec" ]; then
 595                 ExecJob pre "$job_pre_exec" ; ret=$?
 596                 if [ $ret -ne 0 ]; then
 597                         [ $ret -ne 99 ] && count_started=$count_started+1
 598                         echo "Pre-exec command failed, \"$system\" skipped!"
 599                         echo; continue
 600                 fi
 601         fi
 602
 603         # prepare (remote) command ...
 604         if [[ "$backup_type" == "rsync" ]]; then
 605                 cmd="$rsync --archive --timeout=$io_timeout"
 606                 [ "$compress" -ne 0 ] && cmd="$cmd --compress"
 607                 [ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\""
 608                 cmd="$cmd --delete-during --delete-excluded --sparse"
 609                 if [ "$VERBOSE" -gt 0 ]; then
 610                         [ "$rsync_proto" -ge 31 ] \
 611                                 && cmd="$cmd --info=progress2" \
 612                                 || cmd="$cmd --progress"
 613                 fi
 614                 set -f
 615                 if [ "$source_root" = "$default_source_root" ]; then
 616                         for dir in \
 617                                 "/dev/**" \
 618                                 "/media/**" \
 619                                 "/mnt/**" \
 620                                 "/net/**" \
 621                                 "/proc/**" \
 622                                 "/run/**" \
 623                                 "/sys/**" \
 624                                 "/tmp/**" \
 625                                 "/var/cache/apt/**" \
 626                                 "/var/log/**" \
 627                                 "/var/tmp/**" \
 628                         ; do
 629                                 cmd="$cmd --exclude=$dir"
 630                         done
 631                 fi
 632                 [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add"
 633                 for dir in $exclude_dirs_add; do
 634                         cmd="$cmd --exclude=$dir"
 635                 done
 636                 [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add"
 637                 set +f
 638
 639                 [ "$local" -eq 0 ] \
 640                         && cmd="$cmd ${user}@${system}:$source_root $sys_target/" \
 641                         || cmd="$cmd $source_root $sys_target/"
 642         elif [[ "$backup_type" == "scp" ]]; then
 643                 cmd="scp"
 644                 [ "$VERBOSE" -eq 0 ] && cmd="$cmd -q"
 645                 for file in $files; do
 646                         cmd="$cmd ${user}@${system}:$file $sys_target/"
 647                 done
 648         else
 649                 echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!"
 650                 echo; continue
 651         fi
 652
 653         echo "Backing up to \"$sys_target\" ..."
 654         echo -n "Start date: "; date
 655         echo "$cmd"
 656         count_started=$count_started+1
 657         ok=0
 658
 659         if [ "$DRYRUN" -eq 0 ]; then
 660                 stamp_file="$sys_target/.stamp"
 661                 rm -f "$stamp_file"
 662
 663                 # Execute backup command:
 664                 start_t=$(date "+%s")
 665                 $SHELL -c "$cmd"; ret=$?
 666                 end_t=$(date "+%s")
 667
 668                 {
 669                         echo "code=$ret"
 670                         echo "start_t=$start_t"
 671                         echo "end_t=$end_t"
 672                         echo "cmd='$cmd'"
 673                         echo "backup_host='$(hostname -f)'"
 674                         echo "backup_user='$(id -un)'"
 675                 } >"$stamp_file"
 676         else
 677                 echo " *** Trial run, not executing save command!"
 678                 ret=0
 679         fi
 680
 681         if [ $ret -eq 20 ]; then
 682                 echo "Backup of \"$system\" interrupted. Aborting ..."
 683                 GotSignal
 684         fi
 685
 686         echo -n "End date: "; date
 687         if [[ $ret -eq 0 || $ret -eq 24 ]]; then
 688                 [ $ret -eq 24 ] && count_ok_vanished=$count_ok_vanished+1
 689
 690                 echo "System \"$system\" completed with status $ret, OK."
 691                 [ "$DRYRUN" -gt 0 ] || count_ok=$count_ok+1
 692                 ok=1
 693         else
 694                 echo "System \"$system\" completed with ERRORS, code $ret!"
 695         fi
 696
 697         # execute job "post-exec" command, if any
 698         if [ -n "$job_post_exec" ]; then
 699                 ExecJob post "$job_post_exec"
 700         fi
 701
 702         if [ $generations -gt 0 ]; then
 703                 # Update "latest" symlink
 704                 if [ "$DRYRUN" -eq 0 ]; then
 705                         rm -f "$sys_root/latest"
 706                         ln -s "$sys_target" "$sys_root/latest"
 707                 fi
 708                 # Clean up old generations
 709                 declare -i gen_count=$generations+2
 710                 # shellcheck disable=SC2012
 711                 to_delete=$(ls -1t "$sys_root" 2>/dev/null | tail -n+$gen_count | sort)
 712                 if [[ -n "$to_delete" && $ok -eq 1 ]]; then
 713                         [ "$DRYRUN" -eq 0 ] \
 714                                 && echo "Deleting old backup generations (keep $generations) ..." \
 715                                 || echo " *** Trial run, not deleting old generations:"
 716                         for delete in $to_delete; do
 717                                 dir="$sys_root/$delete"
 718                                 if [ ! -e "$dir/.stamp" ]; then
 719                                         echo "Not deleting \"$dir\", not a backup directory!?"
 720                                         continue
 721                                 fi
 722                                 last=$(stat "$dir/.stamp" 2>/dev/null | grep "^Modify: " \
 723                                  | cut -d':' -f2- | cut -d. -f1)
 724                                 # shellcheck disable=SC2086
 725                                 echo "Removing backup from" $last "..."
 726                                 if [ "$DRYRUN" -eq 0 ]; then
 727                                         DeleteSubvolume "$dir"
 728                                         [ $? -eq 0 ] || \
 729                                           echo "Failed to delete \"$dir\"!"
 730                                 fi
 731                         done
 732                         echo -n "Clean up finished: "; date
 733                 elif [ -n "$to_delete" ]; then
 734                         echo "There have been errors, not cleaning up old generations!"
 735                 else
 736                         echo "Nothing to clean up (keep up to $generations generations)."
 737                 fi
 738         fi
 739
 740         destinations="$destinations $target"
 741         echo
 742 done
 743
 744 sync
 745
 746 if [ "$DRYRUN" -eq 0 ]; then
 747         paths=""
 748         paths_zfs=""
 749         # shellcheck disable=SC2086
 750         for dest in $(echo $destinations | sed -e 's/ /\n/g' | sort | uniq); do
 751                 fs=$(GetFS "$dest")
 752                 case $fs in
 753                   "zfs" )
 754                         paths_zfs="$paths_zfs $dest"
 755                         ;;
 756                   *)
 757                         paths="$paths $dest"
 758                 esac
 759         done
 760         if [ -n "$paths" ]; then
 761                 # shellcheck disable=SC2086
 762                 df -h $paths
 763                 echo
 764         fi
 765         if [ -n "$paths_zfs" ]; then
 766                 # shellcheck disable=SC2086
 767                 zfs list $paths_zfs
 768                 echo
 769         fi
 770 fi
 771
 772 CleanUp
 773
 774 echo -n "Done: "; date
 775 echo
 776 [ $count_all -eq 1 ] && s="" || s="s"
 777 [ $count_enabled -eq $count_all ] \
 778         && echo " - $count_all job$s defined (all enabled)," \
 779         || echo " - $count_all job$s defined ($count_enabled enabled),"
 780 [ $count_started -eq 1 ] && s="" || s="s"
 781 echo " - $count_started job$s started,"
 782 echo " - $count_ok done without errors."
 783 echo
 784
 785 if [ $count_started -ne $count_ok ]; then
 786         echo "----->  THERE HAVE BEEN ERRORS!  <-----"
 787         echo
 788         exit 6
 789 elif [ $count_enabled -ne $count_started ]; then
 790         exit 7
 791 fi
 792
 793 # -eof-