diff --git a/helios/external-backup.sh b/helios/external-backup.sh index 964f516..bf7ea80 100644 --- a/helios/external-backup.sh +++ b/helios/external-backup.sh @@ -1,16 +1,16 @@ #!/bin/bash # -# Script zieht per rsync Backups und rotiert Backup-Verzeichnisse +# Script rotiert Backup-Verzeichnisse und zieht per rsync Backups +# http://www.heinlein-support.de # # https://www.heinlein-support.de/howto/backups-und-snapshots-von-linux-servern-mit-rsync-und-ssh # # Zusammengeführt und angepasst durch Tobias Strobel (2021) # -RDIFFEXIT="$(mktemp)" +LOGFILE="$(mktemp)" GOTIFYURL="https://push.strobeto.de/message?token=Anks7VaBnyf7jCB" -HCBACKUPURL="https://hc.strobeto.de/ping/375f20ea-aaa7-4b34-b279-bafc3ff55fed" -HCROTATEURL="https://hc.strobeto.de/ping/f169adf0-a1b3-4823-8fb6-f3a63546bb42" +HCURL="https://hc.strobeto.de/ping/615c0b9b-077d-46b4-913b-b70d911dab27" SERVER="$(hostname)" @@ -39,26 +39,23 @@ clearLED () { finishUp () { # Clean up tempfiles - rm "$RDIFFEXIT" - + rm "$LOGFILE" + # Just to be sure sync -f $DATA_PATH } healthchecksStart () { - # $1 Healthchecks URL - curl -fsS -m 10 --retry 5 -o /dev/null "$1"/start + curl -fsS -m 10 --retry 5 -o /dev/null "$HCURL"/start } healthchecksFinish () { - # $1 Healthchecks URL - # $2 Status Code - # $3 Logs - curl -fsS -m 10 --retry 5 --data-raw "$3" "$1"/"$2" + # $1 Status Code + curl -fsS -m 10 --retry 5 -o /dev/null --data-binary "@$LOGFILE" "$HCURL"/"$1" } notify () { # $1 Action - # $2 Last command succeeded + # $2 Status code # $3 Custom Title # $4 Custom Message # $5 Custom Priority @@ -67,39 +64,21 @@ notify () { NOTIFY_PRIORITY=1 case "$1" in - rotate-start) - NOTIFY_TITLE+="Rotation started" - NOTIFY_MESSAGE="Rotation on external hdd started" - ;; - - rotate-end) - if [ "$2" -eq 0 ]; then - NOTIFY_TITLE+="Rotation finished" - NOTIFY_MESSAGE="Rotation on external hdd finished" - NOTIFY_PRIORITY=5 - else - NOTIFY_TITLE+="Rotation has ERRORS" - NOTIFY_MESSAGE="Exit code $STATUSCODE. See logs attached to healthchecks for more information." - NOTIFY_PRIORITY=8 - fi - ;; - - backup-start) + start) NOTIFY_TITLE+="Backup started" NOTIFY_MESSAGE="Backup to external hdd started" ;; - backup-end) - if [ "$2" -eq 0 ]; then - NOTIFY_TITLE+="Backup finished" - NOTIFY_MESSAGE="Backup to external hdd finished" - NOTIFY_PRIORITY=5 - else - NOTIFY_TITLE+="Backup has ERRORS" - NOTIFY_MESSAGE="Exit code $STATUSCODE. See logs attached to healthchecks for more information." - NOTIFY_PRIORITY=8 - fi + finish) + NOTIFY_TITLE+="Backup finished" + NOTIFY_MESSAGE="Backup to external hdd finished" + NOTIFY_PRIORITY=5 ;; + error) + NOTIFY_TITLE+="Backup has ERRORS" + NOTIFY_MESSAGE="Exit code $STATUSCODE. See logs attached to healthchecks for more information." + NOTIFY_PRIORITY=8 + ;; *) NOTIFY_TITLE+="$3" NOTIFY_MESSAGE="$4" @@ -107,7 +86,7 @@ notify () { ;; esac - curl "$GOTIFYURL" -F "title=$NOTIFY_TITLE" -F "message=$NOTIFY_MESSAGE" -F "priority=$NOTIFY_PRIORITY" + curl -fsS -m 10 --retry 5 -o /dev/null "$GOTIFYURL" -F "title=$NOTIFY_TITLE" -F "message=$NOTIFY_MESSAGE" -F "priority=$NOTIFY_PRIORITY" } checkFreeSpace () { @@ -118,7 +97,8 @@ INODEISFREE=$(df -i /$DATA_PATH | tail -n1 | sed -e "$GETPERCENTAGE") if [ "$KBISFREE" -ge $HDMINFREE ] || [ "$INODEISFREE" -ge $HDMINFREE ] ; then echo "Fatal: Not enough space left for rotating backups!" - logger "Fatal: Not enough space left for rotating backups!" + healthchecksFinish fail + notify custom "NOT_ENOUGH_SPACE_LEFT" "FATAL" "Not enough space left for rotating backups!" exit fi } @@ -126,7 +106,8 @@ fi basicChecks () { if ! [ -d $DATA_PATH ] ; then echo "Fatal: Data path does not exist: $DATA_PATH" - logger "Fatal: Data path does not exist: $DATA_PATH" + healthchecksFinish fail + notify custom "DATA_PATH_NON_EXISTENT" "FATAL" "Data path does not exist: $DATA_PATH" exit fi @@ -134,14 +115,12 @@ basicChecks () { # Ggf. Verzeichnis anlegen if ! [ -d $DATA_PATH/"$SERVER"/daily.0 ] ; then - mkdir -p $DATA_PATH/"$SERVER"/daily.0 + mkdir -p $DATA_PATH/"$SERVER"/daily.0 fi } rotate () { echo "Rotating snapshots of $SERVER..." - healthchecksStart $HCROTATEURL - notify rotate-start # Das hoechste Snapshot abloeschen if [ -d $DATA_PATH/"$SERVER"/daily.7 ] ; then @@ -149,9 +128,9 @@ rotate () { fi # Alle anderen Snapshots eine Nummer nach oben verschieben - for OLD in 6 5 4 3 2 1 ; do + for OLD in 12 11 10 9 8 7 6 5 4 3 2 1 ; do if [ -d $DATA_PATH/"$SERVER"/daily.$OLD ] ; then - NEW=$($OLD + 1) + NEW=$(( OLD + 1 )) # Datum sichern touch $DATA_PATH/.timestamp -r $DATA_PATH/"$SERVER"/daily.$OLD mv $DATA_PATH/"$SERVER"/daily.$OLD $DATA_PATH/"$SERVER"/daily."$NEW" @@ -162,50 +141,62 @@ rotate () { # Snapshot von Level-0 per hardlink-Kopie nach Level-1 kopieren if [ -d $DATA_PATH/"$SERVER"/daily.0 ] ; then - ROTATELOG=$(cp -al $DATA_PATH/"$SERVER"/daily.0 $DATA_PATH/"$SERVER"/daily.1; echo $? > "$RDIFFEXIT") + cp -al $DATA_PATH/"$SERVER"/daily.0 $DATA_PATH/"$SERVER"/daily.1 fi - STATUSCODE=$(cat "$RDIFFEXIT") + STATUSCODE=$? + + if [ "$STATUSCODE" -ne 0 ] ; then + echo "Fatal: rotation finished on $SERVER with errors!" + healthchecksFinish "$STATUSCODE" + notify error "$STATUSCODE" + exit "$STATUSCODE" + fi echo "Finished rotating backups of $SERVER..." - healthchecksFinish $HCROTATEURL "$STATUSCODE" "$ROTATELOG" - notify rotate-end "$STATUSCODE" + healthchecksFinish "$STATUSCODE" + notify error "$STATUSCODE" } backup () { - # Los geht`s: rsync zieht ein Vollbackup echo "Starting rsync backup from $SERVER..." - healthchecksStart $HCBACKUPURL - notify backup-start - BACKUPLOG=$(rsync -avz --numeric-ids -e ssh --delete --delete-excluded \ - --include={"/mnt/storage/nc_data","/mnt/storage/media","/mnt/storage/backup"} \ - --exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/lost+found","/.snapshots"} \ - $SRC_PATH $DATA_PATH/"$SERVER"/daily.0 \ - ; echo $? > "$RDIFFEXIT") + rsync -aAXHh --stats --numeric-ids --noatime --delete --delete-excluded \ + --include={"/mnt/storage","/mnt/storage/nc_data**","/mnt/storage/media**","/mnt/storage/backup**"} \ + --exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/lost+found","/.snapshots"} \ + $SRC_PATH $DATA_PATH/"$SERVER"/daily.0 - STATUSCODE=$(cat "$RDIFFEXIT") + STATUSCODE=$? # Rückgabewert prüfen. # 0 = fehlerfrei, # 24 ist harmlos; tritt auf, wenn während der Laufzeit # von rsync noch (/tmp?) Dateien verändert oder gelöscht wurden. # Alles andere ist fatal -- siehe man (1) rsync - if ! [ "$STATUSCODE" = 24 ] || ! [ "$STATUSCODE" = 0 ] ; then - echo "Fatal: rsync finished $SERVER with errors!" + if [ "$STATUSCODE" -ne 24 ] && [ "$STATUSCODE" -ne 0 ] ; then + echo "Fatal: rsync finished on $SERVER with errors!" + healthchecksFinish "$STATUSCODE" + notify error "$STATUSCODE" + exit "$STATUSCODE" fi # Verzeichnis anfassen, um Backup-Datum zu speichern touch $DATA_PATH/"$SERVER"/daily.0 echo "Finished rsync backup from $SERVER..." - healthchecksFinish $HCBACKUPURL "$STATUSCODE" "$BACKUPLOG" - notify backup-end "$STATUSCODE" } +# Write output to logfile +exec > >(tee -i "${LOGFILE}") +exec 2>&1 + +healthchecksStart +notify start setLED basicChecks backup rotate +healthchecksFinish +notify finish 0 finishUp clearLED \ No newline at end of file