scripts/helios/external-backup.sh

#!/bin/bash
#
# Script rotiert Backup-Verzeichnisse und zieht per rsync Backups
# http://www.heinlein-support.de
#
# https://www.heinlein-support.de/howto/backups-und-snapshots-von-linux-servern-mit-rsync-und-ssh
#
# Zusammengeführt und angepasst durch Tobias Strobel (2021)
#

LOGFILE="$(mktemp)"
SERVER="$(hostname)"

# ### Konfiguration
# Sollen wir pruefen, ob noch ein gewisser Prozentsatz
# an Plattenplatz und Inodes frei ist?
HDMINFREE=90

# Welcher Pfad soll gesichert werden?
SRC_PATH=/

setLED () {
	# Set LED status
	echo heartbeat > "/sys/class/leds/helios64:blue:usb3/trigger"
	echo 1 > "/sys/class/leds/helios64:blue:usb3/brightness"
}

clearLED () {
	# Clear LED status
	echo usbport > "/sys/class/leds/helios64:blue:usb3/trigger"
	echo 0 > "/sys/class/leds/helios64:blue:usb3/brightness"
}

finishUp () {
	# Clean up tempfiles
	rm "$LOGFILE"
	# Just to be sure
	sync -f "$DATA_PATH"
}

healthchecksStart () {
	curl -fsS -m 10 --retry 5 -o /dev/null "$HCURL"/start
}

healthchecksFinish () {
	# $1 Status Code
	curl -fsS -m 10 --retry 5 -o /dev/null --data-binary "@$LOGFILE" "$HCURL"/"$1"
}

notify () {	
	# $1 Action
	# $2 Status code
	# $3 Custom Title
	# $4 Custom Message
	# $5 Custom Priority
	NOTIFY_TITLE="[$SERVER] "
	NOTIFY_MESSAGE=""
	NOTIFY_PRIORITY=1
	case "$1" in

		start)
			NOTIFY_TITLE+="Backup started"
			NOTIFY_MESSAGE="Backup to external hdd started"
		;;

		finish)
			NOTIFY_TITLE+="Backup finished"
			NOTIFY_MESSAGE="Backup to external hdd finished"
			NOTIFY_PRIORITY=5
		;;
		error)
			NOTIFY_TITLE+="Backup has ERRORS"
			NOTIFY_MESSAGE="Exit code $2. See logs attached to healthchecks for more information."
			NOTIFY_PRIORITY=8
		;;		
		*)
			NOTIFY_TITLE+="$3"
			NOTIFY_MESSAGE="$4"
			NOTIFY_PRIORITY="$5"
		;;
	esac

	curl -fsS -m 10 --retry 5 -o /dev/null "$GOTIFYURL" -F "title=$NOTIFY_TITLE" -F "message=$NOTIFY_MESSAGE" -F "priority=$NOTIFY_PRIORITY"
}

checkFreeSpace () {
# Pruefe auf freien Plattenplatz
GETPERCENTAGE='s/.* \([0-9]\{1,3\}\)%.*/\1/'
KBISFREE=$(df /"$DATA_PATH" | tail -n1 | sed -e "$GETPERCENTAGE")
INODEISFREE=$(df -i /"$DATA_PATH" | tail -n1 | sed -e "$GETPERCENTAGE")

if [ "$KBISFREE" -ge $HDMINFREE ] || [ "$INODEISFREE" -ge $HDMINFREE ] ; then
	echo "Fatal: Not enough space left for rotating backups!"
	healthchecksFinish fail
	notify custom "NOT_ENOUGH_SPACE_LEFT" "FATAL" "Not enough space left for rotating backups!"
	exit
fi
}

basicChecks () {
	CONFFILE=~/.config/externalbackup.conf

	if ! [ -f $CONFFILE ] ; then
		cat >>$CONFFILE <<EOF
# GOTIFYURL="https://push.domain.tld/message?token=MYSECRETTOKEN"
GOTIFYURL=""
# HCURL="https://ping.domain.tld/ping/aaaaaaaa-bbbb-cccc-dddddddddddd"
HCURL=""
EOF
	fi

	source $CONFFILE

	if [ -z "$GOTIFYURL" ] || [ -z "$HCURL" ] ; then
		echo "Fatal: GOTIFYURL and HCURL must be set in $CONFFILE. Please edit the file and set correct values, then try again."
		exit 1
	fi

	if [ -z "$1" ] ; then
		echo "A hdd name to backup to must be given as first parameter! (E.g. backupone, for /mnt/backupone)"
		notify custom "NO_HDD_NAME_GIVEN" "FATAL" "No hdd name given to backup to. (E.g. backupone, for /mnt/backupone)"
		exit 1
	fi
	HDDNAME="${1}"

	# Unter welchem Pfad wird gesichert?
	DATA_PATH="/mnt/${HDDNAME}"

	if ! [ -d "$DATA_PATH" ] ; then
		echo "Fatal: Data path does not exist: $DATA_PATH"
		healthchecksFinish fail
		notify custom "DATA_PATH_NON_EXISTENT" "FATAL" "Data path does not exist: $DATA_PATH"
		exit
	fi

	checkFreeSpace

	# Ggf. Verzeichnis anlegen
	if ! [ -d "$DATA_PATH"/"$SERVER"/daily.0 ] ; then
		mkdir -p "$DATA_PATH"/"$SERVER"/daily.0
	fi
}

rotate () {
	echo "Rotating snapshots of $SERVER..."

	# Das hoechste Snapshot abloeschen
	if [ -d "$DATA_PATH"/"$SERVER"/daily.12 ] ; then
			rm -rf "$DATA_PATH"/"$SERVER"/daily.12
	fi

	# Alle anderen Snapshots eine Nummer nach oben verschieben
	for OLD in 12 11 10 9 8 7 6 5 4 3 2 1  ; do
			if [ -d "$DATA_PATH"/"$SERVER"/daily.$OLD ] ; then
				NEW=$(( OLD + 1 ))
			# Datum sichern
			touch "$DATA_PATH"/.timestamp -r "$DATA_PATH"/"$SERVER"/daily.$OLD
					mv "$DATA_PATH"/"$SERVER"/daily.$OLD "$DATA_PATH"/"$SERVER"/daily."$NEW"
			# Datum zurueckspielen
			touch "$DATA_PATH"/"$SERVER"/daily."$NEW" -r "$DATA_PATH"/.timestamp
			fi
	done

	# Snapshot von Level-0 per hardlink-Kopie nach Level-1 kopieren
	if [ -d "$DATA_PATH"/"$SERVER"/daily.0 ] ; then
			cp -al "$DATA_PATH"/"$SERVER"/daily.0 "$DATA_PATH"/"$SERVER"/daily.1
	fi
	
	STATUSCODE=$?
	
	if [ "$STATUSCODE" -ne 0 ] ; then
		echo "Fatal: rotation finished on $SERVER with errors!"
		healthchecksFinish "$STATUSCODE"
		notify error "$STATUSCODE"
		exit "$STATUSCODE"
	fi
	
	echo "Finished rotating backups of $SERVER..."
}

backupNC () {
	NCDIR="/var/www/nextcloud"
	NCOCC="sudo -u www-data php $NCDIR/occ"
	NCDATADIR="$($NCOCC config:system:get datadirectory)"
	NCDUMPSDIR="$NCDATADIR/sqldumps"
	DBPASSWORD="$($NCOCC config:system:get dbpassword)"
	DBNAME="$($NCOCC config:system:get dbname)"
	DBHOST="$($NCOCC config:system:get dbhost)"
	DBUSER="$($NCOCC config:system:get dbuser)"
	DELETEDAYS=15

	if [ ! -f "$NCDIR/config/config.php" ]; then
	  echo "WARN: Nextcloud config not found! Skip Nextcloud backup..."
	  notify custom "NC_CONFIG_NOT_FOUND" "WARN" "Nextcloud config not found! Skip Nextcloud backup..."
	  return 1
	fi

	mkdir -p "$NCDUMPSDIR"
	find "$NCDUMPSDIR"/nextcloud-sqlbkp_* -maxdepth 0 -mmin +$(($DELETEDAYS*60*24)) -exec rm -rvf {} \; > 2&>1

	echo "Starting nextcloud backup..."
	$NCOCC maintenance:mode --on

	echo "Backup Nextcloud DB"
	PGPASSWORD=$DBPASSWORD pg_dump "$DBNAME" -h "$DBHOST" -U "$DBUSER" -f "$NCDUMPSDIR/nextcloud-sqlbkp_$(date +"%Y-%m-%d-%H-%M-%S").bak"


	echo "Backup $NCDATADIR"
	rsync  -aAXHh --stats --numeric-ids --noatime --delete --delete-excluded "$NCDATADIR"/ "$DATA_PATH"/"$SERVER"/daily.0"$NCDATADIR"
			
	STATUSCODE=$?
	
	# Rückgabewert prüfen.
	# 0 = fehlerfrei, 
	# 24 ist harmlos; tritt auf, wenn während der Laufzeit
	# von rsync noch (/tmp?) Dateien verändert oder gelöscht wurden.
	# Alles andere ist fatal -- siehe man (1) rsync
	if [ "$STATUSCODE" -ne 24 ] && [ "$STATUSCODE" -ne 0 ] ; then
		echo "Fatal: rsync for Nextcloud finished on $SERVER with errors!"
		healthchecksFinish "$STATUSCODE"
		notify error "$STATUSCODE"
		$NCOCC maintenance:mode --off
		exit "$STATUSCODE"
	fi
	echo "Finished nextcloud backup..."
	$NCOCC maintenance:mode --off
}

backup () {
	backupNC

	echo "Starting rsync backup from $SERVER..."

	rsync  -aAXHh --stats --numeric-ids --noatime --delete --delete-excluded   \
		--include={"/mnt/storage","/mnt/storage/media**","/mnt/storage/backup**"} \
		--exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/lost+found","/.snapshots"} \
		$SRC_PATH "$DATA_PATH"/"$SERVER"/daily.0
			
	STATUSCODE=$?
	
	# Rückgabewert prüfen.
	# 0 = fehlerfrei, 
	# 24 ist harmlos; tritt auf, wenn während der Laufzeit
	# von rsync noch (/tmp?) Dateien verändert oder gelöscht wurden.
	# Alles andere ist fatal -- siehe man (1) rsync
	if [ "$STATUSCODE" -ne 24 ] && [ "$STATUSCODE" -ne 0 ] ; then
		echo "Fatal: rsync finished on $SERVER with errors!"
		healthchecksFinish "$STATUSCODE"
		notify error "$STATUSCODE"
		exit "$STATUSCODE"
	fi

	# Verzeichnis anfassen, um Backup-Datum zu speichern
	touch "$DATA_PATH"/"$SERVER"/daily.0

	echo "Finished rsync backup from $SERVER..."
}

# Write output to logfile
exec > >(tee -i "${LOGFILE}")
exec 2>&1

basicChecks "${1}"
healthchecksStart
notify start
setLED
backup
rotate
healthchecksFinish
notify finish 0
finishUp
clearLED