bin/backup-audit

   1 #!/bin/bash
   2 #
   3 # backup-script system for cloning systems using rsync
   4 # Copyright (c)2008-2019 Alexander Barton, alex@barton.de
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 # Please read the file COPYING, README and AUTHORS for more information.
  11 #
  12
  13 NAME=$(basename "$0")
  14
  15 VERBOSE=0
  16 QUIET=0
  17
  18 export LC_ALL=C
  19
  20 # Default settings, can be overwritten in backup-script.conf:
  21 [ -d "/usr/local/etc/backup-script.d" ] \
  22         && conf_d="/usr/local/etc/backup-script.d" \
  23         || conf_d="/etc/backup-script.d"
  24
  25 default_backup_type="rsync"
  26 default_files="running-config"
  27 default_generations=0
  28 default_target="/var/backups"
  29
  30 # Set shell options.
  31 shopt -s nullglob
  32
  33 # Search configuration file (last one is used as default!)
  34 for conf in \
  35         "/usr/local/etc/backup-script.conf" \
  36         "/etc/backup-script.conf" \
  37         "${conf_d}/backup-script.conf" \
  38         "/usr/local/etc/backup-script.conf" \
  39 ; do
  40         if [ -r "$conf" ]; then
  41                 # shellcheck source=/dev/null
  42                 source "$conf"
  43                 break
  44         fi
  45 done
  46
  47 Usage() {
  48         echo "Usage: $NAME [-q|--quiet] [-v|--verbose] [<job> [<job> [...]]]"
  49         echo "       $NAME <-d|--dirs> <dir1> <dir2>"
  50         echo
  51         echo "  -d, --dirs      Compare two backup directories (not jobs)."
  52         echo "  -q, --quiet     Quite mode, only list jobs with changes or errors."
  53         echo "  -v, --verbose   Verbose mode, show all checks that are run."
  54         echo
  55         echo "When no <job> is given, all defined jobs are checked."
  56         echo
  57         exit 2
  58 }
  59
  60 BeginDiff() {
  61         echo "Differences in $*:"
  62 }
  63
  64 PipeDiff() {
  65         local line
  66         IFS=
  67         while read -r line; do
  68                 echo -e "     | $line"
  69         done
  70 }
  71
  72 EndDiff() {
  73         :
  74 }
  75
  76 ListDirectory() {
  77         local base_dir="$1"
  78         local dir_name="$2"
  79
  80         local exclude
  81
  82         exclude=' \.$'
  83         if [[ "$dir_name" == "/" ]]; then
  84                 exclude="$exclude"'| \.stamp$| dev$| etc$| proc$| root$| run$| sys$| tmp$'
  85                 exclude="$exclude"'| data$| net$| srv$'
  86                 exclude="$exclude"'| [[:alnum:]_-]+\.log(\.[[:alnum:]]+|)$'
  87                 exclude="$exclude"'| (aquota.user|aquota.group)$'
  88         fi
  89
  90         # shellcheck disable=SC2012
  91         find "$base_dir$dir_name". -maxdepth 1 -printf '%M %10u:%-10g %t %12s  %f\n' 2>/dev/null \
  92                 | LC_ALL=C sort -k 9 | grep -Ev "($exclude)"
  93 }
  94
  95 ListFilesRecursive() {
  96         local base_dir="$1"
  97         local dir_name="$2"
  98
  99         (
 100                 cd "$base_dir" || return 1
 101                 find ".$dir_name" -type f -o -type l | cut -d'/' -f2-
 102         )
 103 }
 104
 105 HandleSystem() {
 106         local fname="$1"
 107
 108         # Set global defaults
 109         local backup_type="$default_backup_type"
 110         local files="$default_files"
 111         local generations="$default_generations"
 112         local local=0
 113         local system="$fname"
 114         local target="$default_target"
 115
 116         # Read in system configuration file
 117         # shellcheck source=/dev/null
 118         source "$f"
 119
 120         target="$target/$(basename "$f")"
 121
 122         [[ -d "$target" ]] || return 0
 123
 124         # System name
 125         [[ "$system" == "$fname" ]] \
 126                 && systxt="\"$system\"" \
 127                 || systxt="\"$fname\" [\"$system\"]"
 128         [[ "$local" -eq 0 ]] \
 129                 && echo "Checking $systxt ..." \
 130                 || echo "Checking $systxt (local system) ..."
 131
 132         # Check if job is disabled
 133         if [[ "$backup_type" == "disabled" ]]; then
 134                 echo "Job is DISABLED and will be skipped."
 135                 echo; return 0
 136         fi
 137
 138         if [ $generations -lt 1 ]; then
 139                 echo "No generations configured, nothing to compare, skipping system!"
 140                 echo; return 1
 141         fi
 142
 143         local latest_d="$target/latest"
 144         if [[ ! -d "$latest_d" || ! -r "$latest_d/.stamp" ]]; then
 145                 echo "Failed to access latest backup generation in \"$latest_d\", skipping system!"
 146                 echo; return 1
 147         fi
 148         echo "Found latest generation in \"$latest_d\"."
 149
 150         declare -i code=-1
 151         # shellcheck source=/dev/null
 152         source "$latest_d/.stamp"
 153
 154         if [[ $code -ne 0 ]]; then
 155                 echo "Warning: Last backup generation had errors, code $code!"
 156         fi
 157
 158         # Search previous generation without errors
 159         local previous_d=""
 160         # shellcheck disable=SC2045
 161         for d in $(ls -1dt "$target/"[0-9]*-[0-9]* 2>/dev/null); do
 162                 [[ -d "$d" && -r "$d/.stamp" ]] || return 0
 163
 164                 declare -i code=-1
 165                 # shellcheck source=/dev/null
 166                 source "$d/.stamp"
 167
 168                 if [[ $code -eq 0 || $code -eq 24 ]]; then
 169                         previous_d="$d"
 170                         break
 171                 fi
 172         done
 173         if [[ -z "$previous_d" || ! -d "$previous_d" || ! -r "$previous_d/.stamp" ]]; then
 174                 echo "Failed to find previous successfull backup generation, skipping system!"
 175                 echo; return 1
 176         fi
 177         echo "Comparing with generation in $previous_d ..."
 178
 179         DiffGenerations "$backup_type" "$previous_d" "$latest_d" "$files"
 180         return_code=$?
 181
 182         echo
 183         return $return_code
 184 }
 185
 186 DiffGenerations() {
 187         local backup_type="$1"
 188         local gen1_d="$2"
 189         local gen2_d="$3"
 190         local files="$4"
 191
 192         local return_code=0
 193
 194         if [[ "$backup_type" == "rsync" ]]; then
 195                 # rsync Backup Type
 196
 197                 for file in \
 198                         /etc/passwd \
 199                         /etc/shadow \
 200                         /etc/group \
 201                         /etc/gshadow \
 202                         \
 203                         /boot/grub/grub.cfg \
 204                         /etc/aliases \
 205                         /etc/bash.bashrc \
 206                         /etc/crontab \
 207                         /etc/debian_version \
 208                         /etc/environment \
 209                         /etc/fstab \
 210                         /etc/hostname \
 211                         /etc/hosts \
 212                         /etc/hosts.allow \
 213                         /etc/hosts.deny \
 214                         /etc/inittab \
 215                         /etc/ld.so.conf \
 216                         /etc/login.defs \
 217                         /etc/machine-id \
 218                         /etc/modules \
 219                         /etc/network/interfaces \
 220                         /etc/networks \
 221                         /etc/nsswitch.conf \
 222                         /etc/profile \
 223                         /etc/rc.local \
 224                         /etc/resolv.conf \
 225                         /etc/services \
 226                         /etc/shells \
 227                         /etc/ssh/sshd_config \
 228                         /etc/sshd_config \
 229                         /etc/sudoers \
 230                         /etc/sysctl.conf \
 231                 ; do
 232                         [[ -r "${gen1_d}${file}" ]] || continue
 233
 234                         [[ $VERBOSE -ne 0 ]] && echo "Checking \"$file\" ..."
 235                         if ! diff -U 3 "${gen1_d}${file}" "${gen2_d}${file}" >"$tmp_diff"; then
 236                                 BeginDiff "\"$file\""
 237                                 tail -n +3 "$tmp_diff" | PipeDiff
 238                                 EndDiff
 239                                 return_code=1
 240                         fi
 241                 done
 242
 243                 for dir in \
 244                         / \
 245                         /etc/cron.d/ \
 246                         /etc/cron.daily/ \
 247                         /etc/cron.hourly/ \
 248                         /etc/cron.monthly/ \
 249                         /etc/cron.weekly/ \
 250                         /etc/init.d/ \
 251                         /etc/sudoers.d/ \
 252                         /var/log/dumps/ \
 253                 ; do
 254                         [[ ! -d "${gen1_d}${dir}" ]] && continue
 255                         [[ ! -d "${gen2_d}${dir}" ]] && continue
 256
 257                         # Make sure that this is a system root; comparing other
 258                         # root folders results in misleading output ...
 259                         [[ "$dir" == "/" && ! -d "${gen1_d}${dir}/etc" ]] && continue
 260
 261                         [[ $VERBOSE -ne 0 ]] && echo "Checking \"$dir\" ..."
 262                         ListDirectory "${gen1_d}" "${dir}" >"$tmp_1"
 263                         ListDirectory "${gen2_d}" "${dir}" >"$tmp_2"
 264                         if ! diff -U 0 "$tmp_1" "$tmp_2" >"$tmp_diff"; then
 265                                 BeginDiff "\"$dir\" directory"
 266                                 tail -n +3 "$tmp_diff" | grep -Ev '^@@ ' | PipeDiff
 267                                 EndDiff
 268                                 return_code=1
 269                         fi
 270                 done
 271
 272                 for dir in \
 273                         /etc/systemd/network/ \
 274                         /etc/systemd/system/ \
 275                         /etc/systemd/user/ \
 276                         /lib/systemd/network/ \
 277                         /lib/systemd/system/ \
 278                         /lib/systemd/user/ \
 279                         /run/systemd/system/ \
 280                         /usr/lib/systemd/network/ \
 281                         /usr/lib/systemd/system/ \
 282                         /usr/lib/systemd/user/ \
 283                 ; do
 284                         [[ ! -d "${gen1_d}${dir}" ]] && continue
 285                         [[ ! -d "${gen2_d}${dir}" ]] && continue
 286
 287                         # Make sure that this is a system root; comparing other
 288                         # root folders results in misleading output ...
 289                         [[ "$dir" == "/" && ! -d "${gen1_d}${dir}/etc" ]] && continue
 290
 291                         [[ $VERBOSE -ne 0 ]] && echo "Checking systemd hierarchy \"$dir\" ..."
 292                         ListFilesRecursive "${gen1_d}" "${dir}" >"$tmp_1"
 293                         ListFilesRecursive "${gen2_d}" "${dir}" >"$tmp_2"
 294                         if ! diff -U 0 "$tmp_1" "$tmp_2" >"$tmp_diff"; then
 295                                 BeginDiff "\"$dir\" directory"
 296                                 tail -n +3 "$tmp_diff" | grep -Ev '^@@ ' | PipeDiff
 297                                 EndDiff
 298                                 return_code=1
 299                         fi
 300                 done
 301
 302                 if [[ -d "${gen1_d}/var/lib/dpkg/info" && -d "${gen2_d}/var/lib/dpkg/info" ]]; then
 303                         [[ $VERBOSE -ne 0 ]] && echo "Checking list of installed packages ..."
 304                         chroot "${gen1_d}" dpkg --get-selections >"$tmp_1" || return 2
 305                         chroot "${gen2_d}" dpkg --get-selections >"$tmp_2" || return 2
 306                         if ! diff -U 0 "$tmp_1" "$tmp_2" >"$tmp_diff"; then
 307                                 BeginDiff "list of installed packages"
 308                                 tail -n +3 "$tmp_diff" | grep -v '^@@ ' | PipeDiff
 309                                 EndDiff
 310                                 return_code=1
 311                         fi
 312                 fi
 313         elif [[ "$backup_type" == "scp" ]]; then
 314                 # scp Backup type
 315                 file=$(basename "$files")
 316                 [[ $VERBOSE -ne 0 ]] && echo "Checking \"$file\" ..."
 317                 if ! diff -U 3 "${gen1_d}/${file}" "${gen2_d}/${file}" >"$tmp_diff"; then
 318                         BeginDiff "\"$file\""
 319                         tail -n +3 "$tmp_diff" | PipeDiff
 320                         EndDiff
 321                         return_code=1
 322                 fi
 323         else
 324                 echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!"
 325                 echo; return 2
 326         fi
 327
 328         return $return_code
 329 }
 330
 331 MkTempFiles() {
 332         tmp_1=$(mktemp "/tmp/$NAME.XXXXXX") || exit 1
 333         tmp_2=$(mktemp "/tmp/$NAME.XXXXXX") || exit 1
 334         tmp_diff=$(mktemp "/tmp/$NAME.XXXXXX") || exit 1
 335         tmp_out=$(mktemp "/tmp/$NAME.XXXXXX") || exit 1
 336 }
 337
 338 CleanUp() {
 339         rm -f "$tmp_1" "$tmp_2" "$tmp_diff" "$tmp_out"
 340 }
 341
 342 while [[ $# -gt 0 ]]; do
 343         case "$1" in
 344           "-d"|"--dirs")
 345                 shift
 346                 [[ $# -eq 2 ]] || Usage
 347                 MkTempFiles
 348                 DiffGenerations "$default_backup_type" "$1" "$2" "$default_files"
 349                 return_code=$?
 350                 CleanUp
 351                 exit $return_code
 352                 ;;
 353           "-q"|"--quiet")
 354                 QUIET=1; shift
 355                 ;;
 356           "-v"|"--verbose")
 357                 VERBOSE=1; shift
 358                 ;;
 359           "-"*)
 360                 Usage
 361                 ;;
 362           *)
 363                 break
 364         esac
 365 done
 366
 367 if [[ $# -ge 1 ]]; then
 368         for s in "$@"; do
 369                 if [ ! -r "${conf_d}/$s" ]; then
 370                         echo "$NAME: Can' read \"${conf_d}/$s\"!"
 371                         exit 1
 372                 fi
 373                 sys+=("${conf_d}/$s")
 374         done
 375 else
 376         sys=("${conf_d}/"*)
 377 fi
 378
 379 MkTempFiles
 380 for f in "${sys[@]}"; do
 381         [[ -r "$f" && -f "$f" ]] || continue
 382
 383         fname=$(basename "$f")
 384         case "$fname" in
 385                 "backup-script.conf"|*.sh)
 386                         continue
 387                         ;;
 388         esac
 389
 390         HandleSystem "$fname" >"$tmp_out" 2>&1; result=$?
 391         [[ $QUIET -eq 0 || $result -ne 0 ]] && cat "$tmp_out"
 392 done
 393 CleanUp
 394
 395 # -eof-