#!/bin/bash
#
# backup-script system for cloning systems using rsync
# Copyright (c)2008-2016 Alexander Barton <alex@barton.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# Please read the file COPYING, README and AUTHORS for more information.
#

NAME=$(basename "$0")
PIDFILE="/var/run/$NAME.pid"

DRYRUN=0
VERBOSE=0
TAG=""
PREPOSTEXEC=1

export LC_ALL=C

declare -i count_all=0
declare -i count_started=0
declare -i count_ok=0
declare -i count_ok_vanished=0
declare -i count_enabled=0

destinations=""

# Default settings, can be overwritten in backup-script.conf:
[ -d "/usr/local/etc/backup-script.d" ] \
	&& conf_d="/usr/local/etc/backup-script.d" \
	|| conf_d="/etc/backup-script.d"
setup_exec=""
pre_exec=""
post_exec=""
default_backup_type="rsync"
default_source_root="/"
default_files="running-config"
default_target="/var/backups"
default_user="root"
default_ssh_args_add=""
default_rsync_args_add=""
default_exclude_args_add=""
default_exclude_dirs_add=""
default_compress=1
default_ping=1
default_local=0
default_generations=0
default_io_timeout="1800"
default_job_pre_exec=""
default_job_post_exec=""
default_tags=""

Usage() {
	echo "Usage: $NAME [<options>] [<job> [<job> [...]]]"
	echo
	echo "  -n, --dry-run       Test run only, don't copy any data."
	echo "  -p, --progress      Show progress, see rsync(1)."
	echo "  -t TAG, --tag TAG   Only run jobs with tag TAG."
	echo "  -x, --no-exec       Don't run global pre-/post-exec commands."
	echo
	echo "When no <job> is given, all defined systems are used."
	echo
	# shellcheck disable=SC2086
	echo -e $config_info
	echo
	exit 2
}

CleanUp() {
	if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then
		echo "Executing \"$post_exec\" ..."
		sh -c $post_exec
		if [ $? -ne 0 ]; then
			echo "Warning: post-exec command failed!"
		fi
		echo
	fi
	rm -f "$PIDFILE"
}

GotSignal() {
	echo
	echo "--> Got signal, cleaning up & aborting ..."
	echo
	CleanUp
	echo -n "Aborted: "; date
	echo
	sleep 3
	exit 9
}

ExecJob() {
	local what="$1"
	local cmd="$2"

	echo "Running job ${what}-exec command ..."
	[ "$local" -eq 0 ] \
		&& cmd="$ssh_cmd ${user}@${system} $cmd"
	echo -n "Start date (${what}-exec): "; date
	echo "$cmd"
	if [ "$DRYRUN" -eq 0 ]; then
		$SHELL -c "$cmd"; local ret=$?
	else
		echo " *** Trial run, not executing ${what}-exec command!"
		ret=0
	fi
	[ $ret -eq 0 ] \
		&& echo "The ${what}-exec command completed with status 0, OK." \
		|| echo "The ${what}-exec command completed with ERRORS, code $ret!"
	return $ret
}

GetFS() {
	local dir="$1"

	while [ -n "$dir" ]; do
		findmnt -fn -o FSTYPE --raw "$dir" 2>/dev/null; local r=$?
		if [ $r -eq 0 ]; then
			return 0
		elif [ $r -eq 127 ]; then
			echo "UNKNOWN"
			return 1
		fi
		dir=$(dirname "$dir") || return 1
	done
}

CreateSubvolume() {
	local volume="$1"
	local fs
	local dir

	dir=$(dirname "$volume")
	fs=$(GetFS "$dir")
	case "$fs" in
	  "btrfs")
		btrfs subvolume create "$volume"  >/dev/null || return 1
		;;
	  "zfs")
		zfs create "$(echo "$volume" | cut -c2-)" || return 1
		;;
	  *)
		echo "CreateSubvolume: Incompatible FS type \"$fs\" on \"$dir\"!"
		return 9
	esac
	return 0
}

CloneSubvolume() {
	local source="$1"
	local volume="$2"
	local snapshot="$3"
	local dir
	local fs
	local link_name

	dir=$(dirname "source")
	fs=$(GetFS "$source")
	case "$fs" in
	  "btrfs")
		btrfs subvolume snapshot "$source" "$snapshot"  >/dev/null || return 1
		;;
	  "zfs")
		zfs snapshot "$snapshot" || return 1
		link_name="$(echo "$snapshot" | cut -d@ -f2-)"
		ln -s \
			"current/.zfs/snapshot/$link_name" \
			"$(dirname "$volume")/$link_name"
		;;
	  *)
		echo "CloneSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
		return 9
	esac
	return 0
}

RenameSubvolume() {
	local source="$1"
	local target="$2"
	local fs

	fs=$(GetFS "$source")
	case "$fs" in
	  "btrfs")
		mv "$source" "$target" || return 1
		;;
	  "zfs")
		zfs rename \
		  "$(echo "$source" | cut -c2-)" \
		  "$(echo "$target" | cut -c2-)" \
			|| return 1
		;;
	  *)
		echo "RenameSubvolume: Incompatible FS type \"$fs\" on \"$source\"!"
		return 9
	esac
	return 0
}

DeleteSubvolume() {
	local volume="$1"
	local fs
	local id
	local snapshot

	fs=$(GetFS "$volume")
	case "$fs" in
	  "btrfs")
		btrfs subvolume delete "$volume" >/dev/null || return 1
		;;
	  "zfs")
		id="$(basename "$volume")"
		if [ -h "$volume" ]; then
			snapshot="$(dirname "$volume")/current@$id"
		else
			snapshot="$volume"
		fi
		zfs destroy -r "$(echo "$snapshot" | cut -c2-)" >/dev/null || return 1
		[ -h "$volume" ] && rm "$volume"
		;;
	  *)
		echo "DeleteSubvolume: Incompatible FS type \"$fs\" on \"$volume\"!"
		return 9
	esac
	return 0
}

Initialize_Last_SysTarget_Snapshot() {
	sys_target="$1"
	unset last
	unset snapshot

	fs=$(GetFS "$sys_target")
	case "$fs" in
	  "btrfs")
		# Search directory of last generation, if any
		# shellcheck disable=SC2012
		last=$(ls -1d "$sys_target"/[0-9]* 2>/dev/null | sort -r | head -n1)
		if [ -n "$last" ]; then
			if [ ! -d "$last" ]; then
				echo "Last snapshot \"$last\" seems not to be a directory!? \"$system\" skipped!"
				echo
				return 1
			fi
		fi
		sys_target="$sys_target/$(date +%Y%m%d-%H%M%S)"
		snapshot="$sys_target"
		;;
	  "zfs")
		# On ZFS, the last generation is always named "current"
		if [ -e "$sys_target/current" ]; then
			last="$sys_target/current"
			if [ "$(uname)" = "Linux" ]; then
				date=$(LC_ALL=C stat "$1" | grep "^Modify: " \
				 | cut -d':' -f2- | cut -d. -f1)
			else
				date=$(LC_ALL=C stat -f "%Sc" "$1")
			fi
			date=$(echo "$date" | sed -e's/^ //g' -e 's/[-:]//g' -e 's/ /-/g')

		else
			last=""
			date="$(date +%Y%m%d-%H%M%S)"
		fi
		snapshot="$(echo "$sys_target/current" | cut -c2-)@$date"
		sys_target="$sys_target/current"
		;;
	  *)
		echo "Initialize_Last_SysTarget_Snapshot: Incompatible FS type \"$fs\" on \"$sys_target\"!"
		return 1
	esac
	return 0
}

# Search configuration file (last one is used as default!)
for conf in \
	"/usr/local/etc/backup-script.conf" \
	"/etc/backup-script.conf" \
	"${conf_d}/backup-script.conf" \
	"/usr/local/etc/backup-script.conf" \
; do
	[ -r "$conf" ] && break
done

# Read in configuration file
config_info="Configuration file is \"$conf\""
if [ -r "$conf" ]; then
	# shellcheck source=/dev/null
	source "$conf"
else
	config_info="${config_info} (not readable, using defaults)"
fi
config_info="${config_info},\nusing \"$conf_d\" as configuration directory."

while [ $# -gt 0 ]; do
	case "$1" in
	  "-n"|"--dry-run")
		DRYRUN=1; shift
		;;
	  "-p"|"--progress")
		VERBOSE=1; shift
		;;
	  "-t"|"--tag")
		shift; TAG="$1"; shift
		[ -n "$TAG" ] || Usage
		;;
	  "-x"|"--no-exec")
		PREPOSTEXEC=0; shift
		;;
	  "-"*)
		Usage
		;;
	  *)
		break
	esac
done

echo -n "Started: "; date
echo -e "$config_info"

# Check rsync and its protocol version
rsync=$(which "rsync" 2>/dev/null)
if [ $? -ne 0 ]; then
	echo "Failed to detect rsync(1)! Is it installed in your \$PATH?"
	exit 1
fi
rsync_proto=$($rsync --version 2>/dev/null | head -n 1 | sed 's/.*  protocol version \([0-9]*\)$/\1/')
if [ $? -ne 0 ]; then
	echo "Failed to detect protocol version of $rsync!"
	exit 1
fi
echo "Rsync command is $rsync, protocol version $rsync_proto."

[[ -n "$TAG" ]] && echo "Running jobs tagged with \"$TAG\"."
echo

if [ $# -ge 1 ]; then
	for s in "$@"; do
		if [ ! -r "${conf_d}/$s" ]; then
			echo "$NAME: Can' read \"${conf_d}/$s\"!"
			exit 3
		fi
		sys+=("${conf_d}/$s")
	done
else
	sys=("${conf_d}/"*)
fi

if [[ -n "$setup_exec" && $PREPOSTEXEC -ne 0 ]]; then
	echo "Executing \"$setup_exec\" ..."
	sh -c $setup_exec
	if [ $? -ne 0 ]; then
		echo "Error: setup command failed!"; echo
		echo "Aborting backup."; echo
		exit 5
	fi
	sleep 2
	echo
fi

trap GotSignal SIGINT SIGTERM

# check and create PID file
if [ -e "$PIDFILE" ]; then
	echo "Lockfile \"$PIDFILE\" already exists."
	echo "Is an other instance still running?"
	echo
	echo -n "Aborted: "; date
	echo
	exit 4
fi
touch "$PIDFILE" 2>/dev/null
if [ $? -ne 0 ]; then
	echo "Warning: can't create PID file \"$PIDFILE\"!"
	echo
else
	echo "$$" >>"$PIDFILE"
fi

if [[ -n "$pre_exec" && $PREPOSTEXEC -ne 0 ]]; then
	echo "Executing \"$pre_exec\" ..."
	sh -c $pre_exec
	if [ $? -ne 0 ]; then
		echo "Error: pre-exec command failed!"; echo
		CleanUp
		echo "Aborting backup."; echo
		exit 5
	fi
	sleep 2
	echo
fi

for f in "${sys[@]}"; do
	[[ -r "$f" && -f "$f" ]] || continue

	fname=$(basename "$f")
	case "$fname" in
		"backup-script.conf"|*.sh)
			continue
			;;
	esac

	# Set global defaults
	system="$fname"
	backup_type="$default_backup_type"
	user="$default_user"
	source_root="$default_source_root"
	files="$default_files"
	target="$default_target"
	ssh_args_add="$default_ssh_args_add"
	rsync_args_add="$default_rsync_args_add"
	exclude_args_add="$default_exclude_args_add"
	exclude_dirs_add="$default_exclude_dirs_add"
	compress="$default_compress"
	ping="$default_ping"
	local="$default_local"
	generations="$default_generations"
	job_pre_exec="$default_job_pre_exec"
	job_post_exec="$default_job_post_exec"
	tags="$default_tags"
	io_timeout="$default_io_timeout"

	# Compatibility with backup-pull(1) script: Save global values ...
	pre_exec_saved="$pre_exec"
	post_exec_saved="$post_exec"

	# Compatibility with backup-pull(1) script: Set defaults
	host=""
	unset source
	unset pre_exec
	unset post_exec

	# Read in system configuration file
	# shellcheck source=/dev/null
	source "$f"

	# Compatibility with backup-pull(1) script: Fix up configuration
	[[ "$system" = "$fname" && -n "$host" ]] \
		&& system="$host"
	[[ "$source_root" = "$default_source_root" && -n "$source" ]] \
		&& source_root="$source"
	[[ -z "$job_pre_exec" && -n "$pre_exec" ]] \
		&& job_pre_exec="$pre_exec"
	[[ -z "$job_post_exec" && -n "$post_exec" ]] \
		&& job_post_exec="$post_exec"

	# Compatibility with backup-pull(1) script: Restore global values ...
	pre_exec="$pre_exec_saved"
	post_exec="$post_exec_saved"

	# Validate configuration
	if [[ "$system" = "localhost" || "$system" = "127.0.0.1" ]]; then
		# Local system
		local=1
		compress=0
	fi

	# Add "NONE" tag when no tags are given in the config file:
	[[ -z "$tags" ]] && tags="NONE"
	# Add "auto-tags":
	[[ "$local" -eq 1 ]] && tags="$tags,LOCAL"
	# Check tags
	if [[ -n "$TAG" && "$TAG" != "ALL" ]]; then
		echo "$tags" | grep -E "(^|,)$TAG(,|$)" >/dev/null 2>&1
		if [ $? -ne 0 ]; then
			if [ "$DRYRUN" -ne 0 ]; then
				echo "Tags of system \"$system\" don't match \"$TAG\": \"$tags\". Skipped."
				echo
			fi
			continue
		fi
	fi

	# Make sure "source_root" ends with a slash ("/")
	case "$source_root" in
	  *"/")
		;;
	  *)
		source_root="$source_root/"
	esac

	# Make sure "target" DOESN'T end with a slash ("/")
	case "$target" in
	  "*/")
		target=$( echo "$target" | sed -e 's/\/$//g' )
		;;
	esac

	[ "$system" = "$fname" ] \
		&& systxt="\"$system\"" \
		|| systxt="\"$fname\" [\"$system\"]"
	[ "$local" -eq 0 ] \
		&& echo "Working on $systxt ..." \
		|| echo "Working on $systxt (local system) ..."

	count_all=$count_all+1

	# Check if job is disabled
	if [ "$backup_type" = "disabled" ]; then
		echo "Job is DISABLED and will be skipped."
		echo; continue
	fi

	count_enabled=$count_enabled+1

	# Check target directory
	if [ -z "$target" ]; then
		echo "No target directory specified for \"$system\"!? Skipped!"
		echo; continue
	fi
	if [ ! -d "$target" ]; then
		echo "Target \"$target\" is not a directory!? \"$system\" skipped!"
		echo; continue
	fi

	sys_target="$target/$fname"
	sys_root="$sys_target"
	if [[ "$DRYRUN" -eq 0 && ! -e "$sys_target" ]]; then
		if [ $generations -gt 0 ]; then
			CreateSubvolume "$sys_target"
		else
			mkdir -p "$sys_target"
		fi
		if [ $? -ne 0 ]; then
			echo "Can't create \"$sys_target\"!? \"$system\" skipped!"
			echo; continue
		fi
	fi

	if [[ "$local" -eq 0 && "$ping" -ne 0 ]]; then
		# Check if system is alive
		ping -c 1 "$system" >/dev/null 2>&1
		if [ $? -ne 0 ]; then
			echo "Host \"$system\" seems not to be alive!? Skipped."
			echo; continue
		fi
		echo "OK, host \"$system\" seems to be alive."
	fi

	if [ $generations -gt 0 ]; then
		# Make sure no old backup is stored in system directory
		if [ -e "$sys_target/.stamp" ]; then
			# There seems to be a genearation-less backup in the
			# target directory!
			echo "Target directory \"$sys_target\" seems to be unclean!? \"$system\" skipped!"
			echo; continue
		fi

		Initialize_Last_SysTarget_Snapshot "$sys_target" || continue

		if [[ -n "$last" && ! -e "$last/.stamp" ]]; then
			# Old backup directory without "stamp file", continue
			echo "Found incomplete snapshot in \"$last\", reusing and renaming it ..."
			RenameSubvolume "$last" "$sys_target"
			if [ $? -ne 0 ]; then
				echo "Failed to rename last snapshot \"$last\" to \"$sys_target\"!? \"$system\" skipped!"
				echo; continue
			fi
		elif [ -n "$last" ]; then
			# Old backup directory found, create new snapshot
			echo "Found last snapshot in \"$last\"."
			if [ "$DRYRUN" -eq 0 ]; then
				CloneSubvolume "$last" "$sys_target" "$snapshot"; r=$?
				if [ $r -ne 0 ]; then
					echo "Can't create snapshot \"$snapshot\" of \"$last\", code $r!? \"$system\" skipped!"
					echo; continue
				fi
				echo "Created new snapshot in \"$snapshot\"."
			else
				echo " *** Trial run, not creating new snapshot in \"$snapshot\"!"
			fi
		else
			# No old backup found, create new subvolume
			if [ "$DRYRUN" -eq 0 ]; then
				CreateSubvolume "$sys_target"; r=$?
				if [ $r -ne 0 ]; then
					echo "Can't create subvolume \"$sys_target\", code $r!? \"$system\" skipped!"
					echo; continue
				fi
				echo "Created new subvolume in \"$sys_target\"."
			else
				echo " *** Trial run, not creating new subvolume \"$sys_target\"!"
			fi
		fi
	fi

	ssh_cmd="ssh"
	[ -n "$ssh_args_add" ] && ssh_cmd="$ssh_cmd $ssh_args_add"

	# execute job "pre-exec" command, if any
	if [ -n "$job_pre_exec" ]; then
		ExecJob pre "$job_pre_exec" ; ret=$?
		if [ $ret -ne 0 ]; then
			[ $ret -ne 99 ] && count_started=$count_started+1
			echo "Pre-exec command failed, \"$system\" skipped!"
			echo; continue
		fi
	fi

	# prepare (remote) command ...
	if [[ "$backup_type" == "rsync" ]]; then
		cmd="$rsync --archive --timeout=$io_timeout"
		[ "$compress" -ne 0 ] && cmd="$cmd --compress"
		[ "$local" -eq 0 ] && cmd="$cmd --rsh=\"$ssh_cmd\""
		cmd="$cmd --delete-during --delete-excluded --sparse"
		if [ "$VERBOSE" -gt 0 ]; then
			[ "$rsync_proto" -ge 31 ] \
				&& cmd="$cmd --info=progress2" \
				|| cmd="$cmd --progress"
		fi
		set -f
		if [ "$source_root" = "$default_source_root" ]; then
			for dir in \
				"/dev/**" \
				"/media/**" \
				"/mnt/**" \
				"/net/**" \
				"/proc/**" \
				"/run/**" \
				"/sys/**" \
				"/tmp/**" \
				"/var/cache/apt/**" \
				"/var/log/**" \
				"/var/tmp/**" \
			; do
				cmd="$cmd --exclude=$dir"
			done
		fi
		[ -n "$exclude_args_add" ] && cmd="$cmd $exclude_args_add"
		for dir in $exclude_dirs_add; do
			cmd="$cmd --exclude=$dir"
		done
		[ -n "$rsync_args_add" ] && cmd="$cmd $rsync_args_add"
		set +f

		[ "$local" -eq 0 ] \
			&& cmd="$cmd ${user}@${system}:$source_root $sys_target/" \
			|| cmd="$cmd $source_root $sys_target/"
	elif [[ "$backup_type" == "scp" ]]; then
		cmd="scp"
		[ "$VERBOSE" -eq 0 ] && cmd="$cmd -q"
		for file in $files; do
			cmd="$cmd ${user}@${system}:$file $sys_target/"
		done
	else
		echo "Backup type \"$backup_type\" undefined, \"$system\" skipped!"
		echo; continue
	fi

	echo "Backing up to \"$sys_target\" ..."
	echo -n "Start date: "; date
	echo "$cmd"
	count_started=$count_started+1
	ok=0

	if [ "$DRYRUN" -eq 0 ]; then
		stamp_file="$sys_target/.stamp"
		rm -f "$stamp_file"

		# Execute backup command:
		start_t=$(date "+%s")
		$SHELL -c "$cmd"; ret=$?
		end_t=$(date "+%s")

		{
			echo "code=$ret"
			echo "start_t=$start_t"
			echo "end_t=$end_t"
			echo "cmd='$cmd'"
			echo "backup_host='$(hostname -f)'"
			echo "backup_user='$(id -un)'"
		} >"$stamp_file"
	else
		echo " *** Trial run, not executing save command!"
		ret=0
	fi

	if [ $ret -eq 20 ]; then
		echo "Backup of \"$system\" interrupted. Aborting ..."
		GotSignal
	fi

	echo -n "End date: "; date
	if [[ $ret -eq 0 || $ret -eq 24 ]]; then
		[ $ret -eq 24 ] && count_ok_vanished=$count_ok_vanished+1

		echo "System \"$system\" completed with status $ret, OK."
		[ "$DRYRUN" -gt 0 ] || count_ok=$count_ok+1
		ok=1
	else
		echo "System \"$system\" completed with ERRORS, code $ret!"
	fi

	# execute job "post-exec" command, if any
	if [ -n "$job_post_exec" ]; then
		ExecJob post "$job_post_exec"
	fi

	if [ $generations -gt 0 ]; then
		# Update "latest" symlink
		if [ "$DRYRUN" -eq 0 ]; then
			rm -f "$sys_root/latest"
			ln -s "$sys_target" "$sys_root/latest"
		fi
		# Clean up old generations
		declare -i gen_count=$generations+2
		# shellcheck disable=SC2012
		to_delete=$(ls -1t "$sys_root" 2>/dev/null | tail -n+$gen_count | sort)
		if [[ -n "$to_delete" && $ok -eq 1 ]]; then
			[ "$DRYRUN" -eq 0 ] \
				&& echo "Deleting old backup generations (keep $generations) ..." \
				|| echo " *** Trial run, not deleting old generations:"
			for delete in $to_delete; do
				dir="$sys_root/$delete"
				if [ ! -e "$dir/.stamp" ]; then
					echo "Not deleting \"$dir\", not a backup directory!?"
					continue
				fi
				last=$(stat "$dir/.stamp" 2>/dev/null | grep "^Modify: " \
				 | cut -d':' -f2- | cut -d. -f1)
				# shellcheck disable=SC2086
				echo "Removing backup from" $last "..."
				if [ "$DRYRUN" -eq 0 ]; then
					DeleteSubvolume "$dir"
					[ $? -eq 0 ] || \
					  echo "Failed to delete \"$dir\"!"
				fi
			done
			echo -n "Clean up finished: "; date
		elif [ -n "$to_delete" ]; then
			echo "There have been errors, not cleaning up old generations!"
		else
			echo "Nothing to clean up (keep up to $generations generations)."
		fi
	fi

	destinations="$destinations $target"
	echo
done

sync

if [ "$DRYRUN" -eq 0 ]; then
	paths=""
	paths_zfs=""
	# shellcheck disable=SC2086
	for dest in $(echo $destinations | sed -e 's/ /\n/g' | sort | uniq); do
		fs=$(GetFS "$dest")
		case $fs in
		  "zfs" )
			paths_zfs="$paths_zfs $dest"
			;;
		  *)
			paths="$paths $dest"
		esac
	done
	if [ -n "$paths" ]; then
		# shellcheck disable=SC2086
		df -h $paths
		echo
	fi
	if [ -n "$paths_zfs" ]; then
		# shellcheck disable=SC2086
		zfs list $paths_zfs
		echo
	fi
fi

CleanUp

echo -n "Done: "; date
echo
[ $count_all -eq 1 ] && s="" || s="s"
[ $count_enabled -eq $count_all ] \
	&& echo " - $count_all job$s defined (all enabled)," \
	|| echo " - $count_all job$s defined ($count_enabled enabled),"
[ $count_started -eq 1 ] && s="" || s="s"
echo " - $count_started job$s started,"
echo " - $count_ok done without errors."
echo

if [ $count_started -ne $count_ok ]; then
	echo "----->  THERE HAVE BEEN ERRORS!  <-----"
	echo
	exit 6
elif [ $count_enabled -ne $count_started ]; then
	exit 7
fi

# -eof-