X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=bin%2Fbackup-script;h=9530410dee0e8d85b7e7b18ad9cefdba4d6cc00f;hb=05cbdc409dd5e4ef1c1d32dcaf15c6baa77e865d;hp=544960d9fedb2f4f207e17c4b496d16da0ce4ee0;hpb=b79f936d8ee57d661d14fb1dd7acab0b8eaed234;p=backup-script.git diff --git a/bin/backup-script b/bin/backup-script index 544960d..9530410 100755 --- a/bin/backup-script +++ b/bin/backup-script @@ -1,7 +1,7 @@ #!/bin/bash # # backup-script system for cloning systems using rsync -# Copyright (c)2008-2015 Alexander Barton +# Copyright (c)2008-2016 Alexander Barton # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,6 +15,8 @@ PIDFILE="/var/run/$NAME.pid" DRYRUN=0 VERBOSE=0 +TAG="" +PREPOSTEXEC=1 export LC_ALL=C @@ -22,6 +24,7 @@ declare -i count_all=0 declare -i count_started=0 declare -i count_ok=0 declare -i count_ok_vanished=0 +declare -i count_enabled=0 destinations="" @@ -29,9 +32,12 @@ destinations="" [ -d "/usr/local/etc/backup-script.d" ] \ && conf_d="/usr/local/etc/backup-script.d" \ || conf_d="/etc/backup-script.d" +setup_exec="" pre_exec="" post_exec="" +default_backup_type="rsync" default_source_root="/" +default_files="running-config" default_target="/var/backups" default_user="root" default_ssh_args_add="" @@ -42,24 +48,29 @@ default_compress=1 default_ping=1 default_local=0 default_generations=0 +default_io_timeout="1800" default_job_pre_exec="" default_job_post_exec="" +default_tags="" Usage() { - echo "Usage: $NAME [] [ [ [...]]]" + echo "Usage: $NAME [] [ [ [...]]]" echo - echo " -p, --progress Show progress, see rsync(1)." - echo " -n, --dry-run Test run only, don't copy any data." + echo " -n, --dry-run Test run only, don't copy any data." + echo " -p, --progress Show progress, see rsync(1)." + echo " -t TAG, --tag TAG Only run jobs with tag TAG." + echo " -x, --no-exec Don't run global pre-/post-exec commands." echo - echo "When no is given, all defined systems are used." + echo "When no is given, all defined systems are used." echo + # shellcheck disable=SC2086 echo -e $config_info echo exit 2 } CleanUp() { - if [ -n "$post_exec" ]; then + if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then echo "Executing \"$post_exec\" ..." sh -c $post_exec if [ $? -ne 0 ]; then @@ -72,7 +83,7 @@ CleanUp() { GotSignal() { echo - echo "--> Got break signal, cleaning up & aborting ..." + echo "--> Got signal, cleaning up & aborting ..." echo CleanUp echo -n "Aborted: "; date @@ -156,7 +167,7 @@ CloneSubvolume() { zfs snapshot "$snapshot" || return 1 link_name="$(echo "$snapshot" | cut -d@ -f2-)" ln -s \ - "$volume/.zfs/snapshot/$link_name" \ + "current/.zfs/snapshot/$link_name" \ "$(dirname "$volume")/$link_name" ;; *) @@ -226,6 +237,7 @@ Initialize_Last_SysTarget_Snapshot() { case "$fs" in "btrfs") # Search directory of last generation, if any + # shellcheck disable=SC2012 last=$(ls -1d "$sys_target"/[0-9]* 2>/dev/null | sort -r | head -n1) if [ -n "$last" ]; then if [ ! -d "$last" ]; then @@ -276,6 +288,7 @@ done # Read in configuration file config_info="Configuration file is \"$conf\"" if [ -r "$conf" ]; then + # shellcheck source=/dev/null source "$conf" else config_info="${config_info} (not readable, using defaults)" @@ -290,6 +303,13 @@ while [ $# -gt 0 ]; do "-p"|"--progress") VERBOSE=1; shift ;; + "-t"|"--tag") + shift; TAG="$1"; shift + [ -n "$TAG" ] || Usage + ;; + "-x"|"--no-exec") + PREPOSTEXEC=0; shift + ;; "-"*) Usage ;; @@ -299,7 +319,7 @@ while [ $# -gt 0 ]; do done echo -n "Started: "; date -echo -e $config_info +echo -e "$config_info" # Check rsync and its protocol version rsync=$(which "rsync" 2>/dev/null) @@ -313,9 +333,9 @@ if [ $? -ne 0 ]; then exit 1 fi echo "Rsync command is $rsync, protocol version $rsync_proto." -echo -trap GotSignal SIGINT +[[ -n "$TAG" ]] && echo "Running jobs tagged with \"$TAG\"." +echo if [ $# -ge 1 ]; then for s in "$@"; do @@ -323,12 +343,26 @@ if [ $# -ge 1 ]; then echo "$NAME: Can' read \"${conf_d}/$s\"!" exit 3 fi - sys="$sys ${conf_d}/$s" + sys+=("${conf_d}/$s") done else - sys="${conf_d}/"* + sys=("${conf_d}/"*) fi +if [[ -n "$setup_exec" && $PREPOSTEXEC -ne 0 ]]; then + echo "Executing \"$setup_exec\" ..." + sh -c $setup_exec + if [ $? -ne 0 ]; then + echo "Error: setup command failed!"; echo + echo "Aborting backup."; echo + exit 5 + fi + sleep 2 + echo +fi + +trap GotSignal SIGINT SIGTERM + # check and create PID file if [ -e "$PIDFILE" ]; then echo "Lockfile \"$PIDFILE\" already exists." @@ -346,7 +380,7 @@ else echo "$$" >>"$PIDFILE" fi -if [ -n "$pre_exec" ]; then +if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then echo "Executing \"$pre_exec\" ..." sh -c $pre_exec if [ $? -ne 0 ]; then @@ -359,8 +393,8 @@ if [ -n "$pre_exec" ]; then echo fi -for f in $sys; do - [ -r "$f" -a -f "$f" ] || continue +for f in "${sys[@]}"; do + [[ -r "$f" && -f "$f" ]] || continue fname=$(basename "$f") case "$fname" in @@ -371,8 +405,10 @@ for f in $sys; do # Set global defaults system="$fname" + backup_type="$default_backup_type" user="$default_user" source_root="$default_source_root" + files="$default_files" target="$default_target" ssh_args_add="$default_ssh_args_add" rsync_args_add="$default_rsync_args_add" @@ -384,28 +420,31 @@ for f in $sys; do generations="$default_generations" job_pre_exec="$default_job_pre_exec" job_post_exec="$default_job_post_exec" + tags="$default_tags" + io_timeout="$default_io_timeout" # Compatibility with backup-pull(1) script: Save global values ... pre_exec_saved="$pre_exec" post_exec_saved="$post_exec" # Compatibility with backup-pull(1) script: Set defaults - unset host + host="" unset source unset pre_exec unset post_exec # Read in system configuration file + # shellcheck source=/dev/null source "$f" # Compatibility with backup-pull(1) script: Fix up configuration - [ "$system" = "$fname" -a -n "$host" ] \ + [[ "$system" = "$fname" && -n "$host" ]] \ && system="$host" - [ "$source_root" = "$default_source_root" -a -n "$source" ] \ + [[ "$source_root" = "$default_source_root" && -n "$source" ]] \ && source_root="$source" - [ -z "$job_pre_exec" -a -n "$pre_exec" ] \ + [[ -z "$job_pre_exec" && -n "$pre_exec" ]] \ && job_pre_exec="$pre_exec" - [ -z "$job_post_exec" -a -n "$post_exec" ] \ + [[ -z "$job_post_exec" && -n "$post_exec" ]] \ && job_post_exec="$post_exec" # Compatibility with backup-pull(1) script: Restore global values ... @@ -413,18 +452,34 @@ for f in $sys; do post_exec="$post_exec_saved" # Validate configuration - if [ "$system" = "localhost" -o "$system" = "127.0.0.1" ]; then + if [[ "$system" = "localhost" || "$system" = "127.0.0.1" ]]; then # Local system local=1 compress=0 fi - # Make sure "source" ends with a slash ("/") - case "$source" in - "*/") + # Add "NONE" tag when no tags are given in the config file: + [[ -z "$tags" ]] && tags="NONE" + # Add "auto-tags": + [[ "$local" -eq 1 ]] && tags="$tags,LOCAL" + # Check tags + if [[ -n "$TAG" && "$TAG" != "ALL" ]]; then + echo "$tags" | grep -E "(^|,)$TAG(,|$)" >/dev/null 2>&1 + if [ $? -ne 0 ]; then + if [ "$DRYRUN" -ne 0 ]; then + echo "Tags of system \"$system\" don't match \"$TAG\": \"$tags\". Skipped." + echo + fi + continue + fi + fi + + # Make sure "source_root" ends with a slash ("/") + case "$source_root" in + *"/") ;; - "*") - source="$source/" + *) + source_root="$source_root/" esac # Make sure "target" DOESN'T end with a slash ("/") @@ -443,6 +498,14 @@ for f in $sys; do count_all=$count_all+1 + # Check if job is disabled + if [ "$backup_type" = "disabled" ]; then + echo "Job is DISABLED and will be skipped." + echo; continue + fi + + count_enabled=$count_enabled+1 + # Check target directory if [ -z "$target" ]; then echo "No target directory specified for \"$system\"!? Skipped!" @@ -455,7 +518,7 @@ for f in $sys; do sys_target="$target/$fname" sys_root="$sys_target" - if [ "$DRYRUN" -eq 0 -a ! -e "$sys_target" ]; then + if [[ "$DRYRUN" -eq 0 && ! -e "$sys_target" ]]; then if [ $generations -gt 0 ]; then CreateSubvolume "$sys_target" else @@ -467,7 +530,7 @@ for f in $sys; do fi fi - if [ "$local" -eq 0 -a "$ping" -ne 0 ]; then + if [[ "$local" -eq 0 && "$ping" -ne 0 ]]; then # Check if system is alive ping -c 1 "$system" >/dev/null 2>&1 if [ $? -ne 0 ]; then @@ -488,7 +551,7 @@ for f in $sys; do Initialize_Last_SysTarget_Snapshot "$sys_target" || continue - if [ -n "$last" -a ! -e "$last/.stamp" ]; then + if [[ -n "$last" && ! -e "$last/.stamp" ]]; then # Old backup directory without "stamp file", continue echo "Found incomplete snapshot in \"$last\", reusing and renaming it ..." RenameSubvolume "$last" "$sys_target" @@ -538,41 +601,54 @@ for f in $sys; do fi # prepare (remote) command ... - cmd="$rsync --archive" - [ "$compress" -ne 0 ] && cmd="$cmd --compress" - [ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\"" - cmd="$cmd --delete --delete-excluded --sparse" - if [ "$VERBOSE" -gt 0 ]; then - [ "$rsync_proto" -ge 31 ] \ - && cmd="$cmd --info=progress2" \ - || cmd="$cmd --progress" - fi - if [ "$source_root" = "$default_source_root" ]; then - for dir in \ - "/dev/**" \ - "/media/**" \ - "/mnt/**" \ - "/net/**" \ - "/proc/**" \ - "/run/**" \ - "/sys/**" \ - "/tmp/**" \ - "/var/cache/apt/**" \ - "/var/log/**" \ - "/var/tmp/**" \ - ; do + if [[ "$backup_type" == "rsync" ]]; then + cmd="$rsync --archive --timeout=$io_timeout" + [ "$compress" -ne 0 ] && cmd="$cmd --compress" + [ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\"" + cmd="$cmd --delete-during --delete-excluded --sparse" + if [ "$VERBOSE" -gt 0 ]; then + [ "$rsync_proto" -ge 31 ] \ + && cmd="$cmd --info=progress2" \ + || cmd="$cmd --progress" + fi + set -f + if [ "$source_root" = "$default_source_root" ]; then + for dir in \ + "/dev/**" \ + "/media/**" \ + "/mnt/**" \ + "/net/**" \ + "/proc/**" \ + "/run/**" \ + "/sys/**" \ + "/tmp/**" \ + "/var/cache/apt/**" \ + "/var/log/**" \ + "/var/tmp/**" \ + ; do + cmd="$cmd --exclude=$dir" + done + fi + [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add" + for dir in $exclude_dirs_add; do cmd="$cmd --exclude=$dir" done + [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add" + set +f + + [ "$local" -eq 0 ] \ + && cmd="$cmd ${user}@${system}:$source_root $sys_target/" \ + || cmd="$cmd $source_root $sys_target/" + elif [[ "$backup_type" == "scp" ]]; then + cmd="scp" + [ "$VERBOSE" -eq 0 ] && cmd="$cmd -q" + for file in $files; do + cmd="$cmd ${user}@${system}:$file $sys_target/" + done + else + echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!" + echo; continue fi - [ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add" - for dir in $exclude_dirs_add; do - cmd="$cmd --exclude=$dir" - done - [ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add" - - [ "$local" -eq 0 ] \ - && cmd="$cmd ${user}@${system}:$source_root $sys_target/" \ - || cmd="$cmd $source_root $sys_target/" echo "Backing up to \"$sys_target\" ..." echo -n "Start date: "; date @@ -581,11 +657,24 @@ for f in $sys; do ok=0 if [ "$DRYRUN" -eq 0 ]; then - rm -f "$sys_target/.stamp" + stamp_file="$sys_target/.stamp" + rm -f "$stamp_file" + + # Execute backup command: + start_t=$(date "+%s") $SHELL -c "$cmd"; ret=$? - echo "code=$ret" >"$sys_target/.stamp" + end_t=$(date "+%s") + + { + echo "code=$ret" + echo "start_t=$start_t" + echo "end_t=$end_t" + echo "cmd='$cmd'" + echo "backup_host='$(hostname -f)'" + echo "backup_user='$(id -un)'" + } >"$stamp_file" else - echo " *** Trial run, not executing synchronization command!" + echo " *** Trial run, not executing save command!" ret=0 fi @@ -595,7 +684,7 @@ for f in $sys; do fi echo -n "End date: "; date - if [ $ret -eq 0 -o $ret -eq 24 ]; then + if [[ $ret -eq 0 || $ret -eq 24 ]]; then [ $ret -eq 24 ] && count_ok_vanished=$count_ok_vanished+1 echo "System \"$system\" completed with status $ret, OK." @@ -618,8 +707,9 @@ for f in $sys; do fi # Clean up old generations declare -i gen_count=$generations+2 + # shellcheck disable=SC2012 to_delete=$(ls -1t "$sys_root" 2>/dev/null | tail -n+$gen_count | sort) - if [ -n "$to_delete" -a $ok -eq 1 ]; then + if [[ -n "$to_delete" && $ok -eq 1 ]]; then [ "$DRYRUN" -eq 0 ] \ && echo "Deleting old backup generations (keep $generations) ..." \ || echo " *** Trial run, not deleting old generations:" @@ -631,6 +721,7 @@ for f in $sys; do fi last=$(stat "$dir/.stamp" 2>/dev/null | grep "^Modify: " \ | cut -d':' -f2- | cut -d. -f1) + # shellcheck disable=SC2086 echo "Removing backup from" $last "..." if [ "$DRYRUN" -eq 0 ]; then DeleteSubvolume "$dir" @@ -642,7 +733,7 @@ for f in $sys; do elif [ -n "$to_delete" ]; then echo "There have been errors, not cleaning up old generations!" else - echo "Nothing to clean up." + echo "Nothing to clean up (keep up to $generations generations)." fi fi @@ -655,6 +746,7 @@ sync if [ "$DRYRUN" -eq 0 ]; then paths="" paths_zfs="" + # shellcheck disable=SC2086 for dest in $(echo $destinations | sed -e 's/ /\n/g' | sort | uniq); do fs=$(GetFS "$dest") case $fs in @@ -666,10 +758,12 @@ if [ "$DRYRUN" -eq 0 ]; then esac done if [ -n "$paths" ]; then + # shellcheck disable=SC2086 df -h $paths echo fi if [ -n "$paths_zfs" ]; then + # shellcheck disable=SC2086 zfs list $paths_zfs echo fi @@ -680,7 +774,9 @@ CleanUp echo -n "Done: "; date echo [ $count_all -eq 1 ] && s="" || s="s" -echo " - $count_all job$s defined," +[ $count_enabled -eq $count_all ] \ + && echo " - $count_all job$s defined (all enabled)," \ + || echo " - $count_all job$s defined ($count_enabled enabled)," [ $count_started -eq 1 ] && s="" || s="s" echo " - $count_started job$s started," echo " - $count_ok done without errors." @@ -690,7 +786,7 @@ if [ $count_started -ne $count_ok ]; then echo "-----> THERE HAVE BEEN ERRORS! <-----" echo exit 6 -elif [ $count_all -ne $count_started ]; then +elif [ $count_enabled -ne $count_started ]; then exit 7 fi