#!/bin/bash
#
# Script rotiert Backup-Verzeichnisse und zieht per rsync Backups
# http://www.heinlein-support.de
#
# https://www.heinlein-support.de/howto/backups-und-snapshots-von-linux-servern-mit-rsync-und-ssh
#
# Zusammengeführt und angepasst durch Tobias Strobel (2021)
#

LOGFILE="$(mktemp)"
GOTIFYURL="https://push.strobeto.de/message?token=Anks7VaBnyf7jCB"
HCURL="https://hc.strobeto.de/ping/615c0b9b-077d-46b4-913b-b70d911dab27"

SERVER="$(hostname)"

# ### Konfiguration
# Sollen wir pruefen, ob noch ein gewisser Prozentsatz
# an Plattenplatz und Inodes frei ist?
HDMINFREE=90

# Welcher Pfad soll gesichert werden?
SRC_PATH=/

# Unter welchem Pfad wird gesichert?
DATA_PATH=/mnt/backupone

setLED () {
	# Set LED status
	echo heartbeat > "/sys/class/leds/helios64:blue:usb3/trigger"
	echo 1 > "/sys/class/leds/helios64:blue:usb3/brightness"
}

clearLED () {
	# Clear LED status
	echo usbport > "/sys/class/leds/helios64:blue:usb3/trigger"
	echo 0 > "/sys/class/leds/helios64:blue:usb3/brightness"
}

finishUp () {
	# Clean up tempfiles
	rm "$LOGFILE"
	# Just to be sure
	sync -f $DATA_PATH
}

healthchecksStart () {
	curl -fsS -m 10 --retry 5 -o /dev/null "$HCURL"/start
}

healthchecksFinish () {
	# $1 Status Code
	curl -fsS -m 10 --retry 5 -o /dev/null --data-binary "@$LOGFILE" "$HCURL"/"$1"
}

notify () {	
	# $1 Action
	# $2 Status code
	# $3 Custom Title
	# $4 Custom Message
	# $5 Custom Priority
	NOTIFY_TITLE="[$SERVER] "
	NOTIFY_MESSAGE=""
	NOTIFY_PRIORITY=1
	case "$1" in

		start)
			NOTIFY_TITLE+="Backup started"
			NOTIFY_MESSAGE="Backup to external hdd started"
		;;

		finish)
			NOTIFY_TITLE+="Backup finished"
			NOTIFY_MESSAGE="Backup to external hdd finished"
			NOTIFY_PRIORITY=5
		;;
		error)
			NOTIFY_TITLE+="Backup has ERRORS"
			NOTIFY_MESSAGE="Exit code $STATUSCODE. See logs attached to healthchecks for more information."
			NOTIFY_PRIORITY=8
		;;		
		*)
			NOTIFY_TITLE+="$3"
			NOTIFY_MESSAGE="$4"
			NOTIFY_PRIORITY="$5"
		;;
	esac

	curl -fsS -m 10 --retry 5 -o /dev/null "$GOTIFYURL" -F "title=$NOTIFY_TITLE" -F "message=$NOTIFY_MESSAGE" -F "priority=$NOTIFY_PRIORITY"
}

checkFreeSpace () {
# Pruefe auf freien Plattenplatz
GETPERCENTAGE='s/.* \([0-9]\{1,3\}\)%.*/\1/'
KBISFREE=$(df /$DATA_PATH | tail -n1 | sed -e "$GETPERCENTAGE")
INODEISFREE=$(df -i /$DATA_PATH | tail -n1 | sed -e "$GETPERCENTAGE")

if [ "$KBISFREE" -ge $HDMINFREE ] || [ "$INODEISFREE" -ge $HDMINFREE ] ; then
	echo "Fatal: Not enough space left for rotating backups!"
	healthchecksFinish fail
	notify custom "NOT_ENOUGH_SPACE_LEFT" "FATAL" "Not enough space left for rotating backups!"
	exit
fi
}

basicChecks () {
	if ! [ -d $DATA_PATH ] ; then
		echo "Fatal: Data path does not exist: $DATA_PATH"
		healthchecksFinish fail
		notify custom "DATA_PATH_NON_EXISTENT" "FATAL" "Data path does not exist: $DATA_PATH"
		exit
	fi

	checkFreeSpace

	# Ggf. Verzeichnis anlegen
	if ! [ -d $DATA_PATH/"$SERVER"/daily.0 ] ; then
		mkdir -p $DATA_PATH/"$SERVER"/daily.0
	fi
}

rotate () {
	echo "Rotating snapshots of $SERVER..."

	# Das hoechste Snapshot abloeschen
	if [ -d $DATA_PATH/"$SERVER"/daily.12 ] ; then
			rm -rf $DATA_PATH/"$SERVER"/daily.12
	fi

	# Alle anderen Snapshots eine Nummer nach oben verschieben
	for OLD in 12 11 10 9 8 7 6 5 4 3 2 1  ; do
			if [ -d $DATA_PATH/"$SERVER"/daily.$OLD ] ; then
				NEW=$(( OLD + 1 ))
			# Datum sichern
			touch $DATA_PATH/.timestamp -r $DATA_PATH/"$SERVER"/daily.$OLD
					mv $DATA_PATH/"$SERVER"/daily.$OLD $DATA_PATH/"$SERVER"/daily."$NEW"
			# Datum zurueckspielen
			touch $DATA_PATH/"$SERVER"/daily."$NEW" -r $DATA_PATH/.timestamp
			fi
	done

	# Snapshot von Level-0 per hardlink-Kopie nach Level-1 kopieren
	if [ -d $DATA_PATH/"$SERVER"/daily.0 ] ; then
			cp -al $DATA_PATH/"$SERVER"/daily.0 $DATA_PATH/"$SERVER"/daily.1
	fi
	
	STATUSCODE=$?
	
	if [ "$STATUSCODE" -ne 0 ] ; then
		echo "Fatal: rotation finished on $SERVER with errors!"
		healthchecksFinish "$STATUSCODE"
		notify error "$STATUSCODE"
		exit "$STATUSCODE"
	fi
	
	echo "Finished rotating backups of $SERVER..."
}

backup () {
	echo "Starting rsync backup from $SERVER..."

	rsync  -aAXHh --stats --numeric-ids --noatime --delete --delete-excluded   \
		--include={"/mnt/storage","/mnt/storage/nc_data**","/mnt/storage/media**","/mnt/storage/backup**"} \
		--exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/lost+found","/.snapshots"} \
		$SRC_PATH $DATA_PATH/"$SERVER"/daily.0
			
	STATUSCODE=$?
	
	# Rückgabewert prüfen.
	# 0 = fehlerfrei, 
	# 24 ist harmlos; tritt auf, wenn während der Laufzeit
	# von rsync noch (/tmp?) Dateien verändert oder gelöscht wurden.
	# Alles andere ist fatal -- siehe man (1) rsync
	if [ "$STATUSCODE" -ne 24 ] && [ "$STATUSCODE" -ne 0 ] ; then
		echo "Fatal: rsync finished on $SERVER with errors!"
		healthchecksFinish "$STATUSCODE"
		notify error "$STATUSCODE"
		exit "$STATUSCODE"
	fi

	# Verzeichnis anfassen, um Backup-Datum zu speichern
	touch $DATA_PATH/"$SERVER"/daily.0

	echo "Finished rsync backup from $SERVER..."
}

# Write output to logfile
exec > >(tee -i "${LOGFILE}")
exec 2>&1

healthchecksStart
notify start
setLED
basicChecks
backup
rotate
healthchecksFinish
notify finish 0
finishUp
clearLED