testing backups

hid some messages behind VERBOSE option
unified date formats of tars

5 files changed, 98 insertions, 34 deletions

diff --git a/backends/borg.sh b/backends/borg.sh
index 0b28619..5d7b2c7 100644
--- a/backends/borg.sh
+++ b/backends/borg.sh
@@ -16,7 +16,7 @@ function borg_create_backup() {
 
         trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
 
-        echo "borg: starting backup to \"$backup_dir\""
+        echo_debug "borg: starting backup to \"$backup_dir\""
 
         borg create                         \
             "${backup_dir}::${BACKUP_NAME}_$(date +'%F_%H-%M-%S')" \
@@ -27,7 +27,7 @@ function borg_create_backup() {
 
         local backup_exit=$?
 
-        echo "borg: pruning repository at \"$backup_dir\""
+        echo_debug "borg: pruning repository at \"$backup_dir\""
 
         borg prune                          \
             --prefix '{hostname}-'          \
@@ -45,7 +45,7 @@ function borg_create_backup() {
         retcode=$(( global_exit > retcode ? global_exit : retcode ))
 
         if [ ${global_exit} -eq 0 ]; then
-            echo "borg: backup and prune finished successfully"
+            echo_debug "borg: backup and prune finished successfully"
         elif [ ${global_exit} -eq 1 ]; then
             echo "borg: backup and/or prune finished with warnings"
         else
@@ -75,6 +75,7 @@ function borg_restore() {
     export BORG_PASSCOMMAND="$BACKUP_PASSCOMMAND"
     local remote="$1"
     local snapshot="$2"
+	local dest="$3"
 
     export BORG_REPO="$remote"
     borg extract "${remote}::${snapshot}"
diff --git a/backends/bup.sh b/backends/bup.sh
index b57bd86..50c2b86 100644
--- a/backends/bup.sh
+++ b/backends/bup.sh
@@ -15,14 +15,14 @@ function bup_init() {
     bup -d "$(bup_local)" index "$WORLD_NAME"
     local status=$?
     if [ $status -ne 0 ]; then
-        echo "bup: no local repo found, creating..."
+        echo_debug "bup: no local repo found, creating..."
         bup -d "$(bup_local)" init -r "$(bup_local)"
-        echo "bup: created local repo at $(bup_local)"
+        echo_debug "bup: created local repo at $(bup_local)"
     fi
 }
 
 function bup_create_backup() {
-    echo "bup: backup started"
+    echo_debug "bup: backup started"
 
     bup -d "$(bup_local)" index "$WORLD_NAME"
 
@@ -31,16 +31,17 @@ function bup_create_backup() {
     local retcode=1
     for backup_dir in ${BACKUP_DIRS[*]}
     do
-        echo "bup: backing up to \"$backup_dir\""
+        echo_debug "bup: backing up to \"$backup_dir\""
         # try to save to remote
         bup -d "$(bup_local)" save -r "$backup_dir" -n "$BACKUP_NAME" "$WORLD_NAME"
 		local status=$?
         # if failed - reinit remote and try again
         if [ $status -ne 0 ]; then
-            echo "bup: failed backing up to \"$backup_dir\", reinitializing remote..."
+            echo_debug "bup: failed backing up to \"$backup_dir\", reinitializing remote..."
             bup -d "$(bup_local)" init -r "$backup_dir"
-            if [ $? -ne 0 ]; then
-                echo "bup: created remote at \"$backup_dir\""
+			status=$?
+            if [ $status -ne 0 ]; then
+                echo_debug "bup: created remote at \"$backup_dir\""
                 bup -d "$(bup_local)" save -r "$backup_dir" -n "$BACKUP_NAME" "$WORLD_NAME"
             else
                 echo "bup: failed to make remote at \"$backup_dir\", moving on"
@@ -73,5 +74,6 @@ function bup_ls_all() {
 function bup_restore() {
     local remote="$1"
     local snapshot="$2"
-    bup -d "$(bup_local)" restore -r "$remote" "$BACKUP_NAME/$snapshot/$PWD/."
+	local dest="$3"
+    bup -d "$(bup_local)" restore -r "$remote" --outdir "$dest" "$BACKUP_NAME/$snapshot/$PWD/."
 }
diff --git a/backends/tar.sh b/backends/tar.sh
index f7ea86e..00018c6 100644
--- a/backends/tar.sh
+++ b/backends/tar.sh
@@ -5,12 +5,12 @@ function tar_init() {
 
 # TODO: Make default .tar with optional bup
 function tar_create_backup() {
-    echo "tar: backing up..."
+    echo_debug "tar: backing up..."
 
 	local status
 
     # save world to a temporary archive
-    local archname="/tmp/${BACKUP_NAME}_`date +%FT%H%M%S%z`.tar.gz"
+    local archname="/tmp/${BACKUP_NAME}_`date +%F_%H-%M-%S`.tar.gz"
     tar -czf "$archname" "./$WORLD_NAME"
 	status=$?
     if [ $status -ne 0 ]; then
@@ -18,14 +18,14 @@ function tar_create_backup() {
         rm "$archname" #remove (probably faulty) archive
         return 1
     fi
-    echo "tar: world saved to $archname, pushing it to backup directories..."
+    echo_debug "tar: world saved to $archname, pushing it to backup directories..."
 
 	# 0 if could save to at least one backup dir
 	# TODO: make more strict?
     local retcode=1
     for backup_dir in ${BACKUP_DIRS[*]}
     do
-        echo "tar: pushing to \"$backup_dir\""
+        echo_debug "tar: pushing to \"$backup_dir\""
         # scp acts as cp for local destination directories
         scp "$archname" "$backup_dir/"
 		status=$?
@@ -51,22 +51,23 @@ function tar_ls_dir() {
         local remote="$(echo "$backup_dir" | cut -d: -f1)"
         local remote_dir="$(echo "$backup_dir" | cut -d: -f2)"
         ssh "$remote" "ls -1 $remote_dir" | grep "tar.gz" | sort -r
-    else
+	else
         ls -1 "$backup_dir" | grep "tar.gz" | sort -r
-    fi
+	fi
 }
 
 function tar_ls_all() {
     for backup_dir in ${BACKUP_DIRS[*]}
     do
         echo "tar: backups in ${backup_dir}:"
-        tar_ls_remote "$backup_dir"
+        tar_ls_dir "$backup_dir"
     done
 }
 
 function tar_restore() {
     local remote="$1"
     local snapshot="$2"
+	local dest="$3"
 	local status
 
     scp "$remote/$snapshot" "/tmp/"
@@ -76,5 +77,5 @@ function tar_restore() {
         return 1
     fi
 
-    tar -xzf "/tmp/$snapshot"
+    tar -xzf "/tmp/$snapshot" -C "$dest"
 }
diff --git a/server.sh b/server.sh
index ad1a7d8..1f4f8e1 100755
--- a/server.sh
+++ b/server.sh
@@ -12,6 +12,12 @@ source "backends/tar.sh"
 source "backends/bup.sh"
 source "backends/borg.sh"
 
+function echo_debug() {
+	if [ $VERBOSE -ne 0 ]; then
+		echo "$1"
+	fi
+}
+
 function backup_hook_example {
 	bup -d $CUR_BACK_DIR ls -l $BACKUP_NAME/latest/var/minecraft
 }
@@ -111,6 +117,7 @@ function players_online() {
 }
 
 function init_backup() {
+	# even though bup and borg are smart, they will not create a path for a repo
 	for backup_dir in ${BACKUP_DIRS[*]}
 	do
 		if [[ $backup_dir == *:* ]]; then
@@ -131,6 +138,42 @@ function init_backup() {
 	fi
 }
 
+function same_world() {
+	delta=$(diff -r "$1" "$2")
+	if [ -z "$delta" ] ; then
+		return 0
+	fi
+	return 1
+}
+
+# checking if latest snapshots are the same as the current world
+function test_backup_integrity() {
+	local retcode=0
+	for backup_dir in ${BACKUP_DIRS[*]}
+	do
+		local serverdir="$PWD"
+		local tmpdir=$(mktemp -d);
+
+		# restore most recent backup to a temporary dir
+		if ! server_restore "$serverdir/$backup_dir" 0 "$tmpdir" ; then
+			echo "Failed to get latest snapshot from \"$backup_dir\""
+			retcode=1
+		elif ! same_world "$WORLD_NAME" "$tmpdir/$WORLD_NAME" ; then
+			echo "Latest backup from \"$backup_dir\" differs from current world!"
+			retcode=1
+		fi
+
+		rm -r "$tmpdir"
+	done
+
+	if [ $retcode -ne 0 ] ; then
+		echo "Backup integrity check: FAILED"
+		return 1
+	fi
+	echo "Backup integrity check: OK"
+	return 0
+}
+
 function create_backup() {
 	init_backup
 
@@ -141,6 +184,8 @@ function create_backup() {
 	else
 		tar_create_backup
 	fi
+
+	test_backup_integrity
 }
 
 function server_backup_safe() {
@@ -255,13 +300,13 @@ function is_in() {
 function server_restore() {
 	local backup_dir
 	local snapshot_index
-	local dest
+	local dest="$PWD"
 
+	# parameters are only used for testing backups, so thorough checks are not needed
 	if [ $# -ge 2 ]; then
 		backup_dir="$1"
 		snapshot_index=$2
 	fi
-
 	if [ $# -eq 3 ]; then
 		dest="$3"
 	fi
@@ -270,6 +315,8 @@ function server_restore() {
 		echo "No backup directories found, abort"
 		return 1
 	fi
+
+
 	if [ -z $backup_dir ]; then
 		echo "From where get the snapshot?"
 		backup_dir="$(choose_from "${BACKUP_DIRS[@]}")"
@@ -281,6 +328,7 @@ function server_restore() {
 		return 1
 	fi
 
+
 	local snapshots=$(
 		if [ $BACKUP_BACKEND = "bup" ]; then
 			bup_ls_dir "$backup_dir"
@@ -297,6 +345,7 @@ function server_restore() {
 	# convert multiline string to bash array
 	snapshots=($(echo "$snapshots"))
 
+
 	local snapshot
 	if [ -z $snapshot_index ]; then
 		echo "Select which snapshot to restore"
@@ -309,29 +358,37 @@ function server_restore() {
 		return 1
 	fi
 
-	echo "Restoring snapshot \"$snapshot\" from \"$backup_dir\""
 
-	local oldworld_name=""
-	if [[ -d "$WORLD_NAME" ]]; then
+	echo_debug "Restoring snapshot \"$snapshot\" from \"$backup_dir\""
+
+
+	# if we restore to PWD, we will overwrite the current world, which might be harmful
+	local oldworld_name
+	if [ "$dest" = "$PWD" ] && [[ -d "$WORLD_NAME" ]]; then
 		echo -n "Preserving old world: "
 		oldworld_name="${WORLD_NAME}.old.$(date +'%F_%H-%M-%S')"
 		mv -v "$PWD/$WORLD_NAME" "$PWD/$oldworld_name"
 	fi
 
+
 	if [ $BACKUP_BACKEND = "bup" ]; then
-		bup_restore "$backup_dir" "$snapshot"
+		bup_restore "$backup_dir" "$snapshot" "$dest"
 	elif [ $BACKUP_BACKEND = "borg" ]; then
-		borg_restore "$backup_dir" "$snapshot"
+		borg_restore "$backup_dir" "$snapshot" "$dest"
 	else
-		tar_restore "$backup_dir" "$snapshot"
+		tar_restore "$backup_dir" "$snapshot" "$dest"
 	fi
 	local status=$?
-	if [ $status -ne 0 ]; then
+
+
+	# if we preseved the current world, but failed to restore the snapshot
+	if [ ! -z ${oldworld_name+x} ] && [ $status -ne 0 ]; then
 		echo "Failed to restore snapshot, putting old world back where it was:"
 		rm -rv "$PWD/$WORLD_NAME"
 		mv -v "$PWD/$oldworld_name" "$PWD/$WORLD_NAME"
 		return 1
 	fi
+
 	echo "Snapshot restored"
 
 	return 0
diff --git a/serverconf.sh b/serverconf.sh
index 3748541..a8f02a6 100644
--- a/serverconf.sh
+++ b/serverconf.sh
@@ -2,6 +2,8 @@
 # configuration file for server.sh minecraft server
 # management script
 
+VERBOSE=0
+
 #CONFIG
 JRE_JAVA="java"
 JVM_ARGS="-Xms4096M -Xmx6144M"
@@ -21,16 +23,17 @@ BACKUP_NAME="${WORLD_NAME}_backup"
 LOGFILE="logs/latest.log"
 PIDFILE="server-screen.pid"
 # if not bup or borg, uses tar by default
-#BACKUP_BACKEND="tar"
+BACKUP_BACKEND="tar"
 #BACKUP_BACKEND="bup"
-BACKUP_BACKEND="borg"
+#BACKUP_BACKEND="borg"
 
 #Constants
 CUR_YEAR=`date +"%Y"`
 
-BACKUP_DIRS=(".bak/$CUR_YEAR" "user@backupserver:/path/to/backup/$CUR_YEAR")
+# IMPORTANT: local paths must be absolute!
+BACKUP_DIRS=( "$PWD/.bak/$CUR_YEAR" "user@backupserver:/path/to/backup/$CUR_YEAR" )
 
-# borg repository could be password protected
+# borg repositories are password protected by default
 # to avoid having to manually type password, borg can run a command that should echo a password
-#BACKUP_PASSCOMMAND="echo superstrongpassword"
-#BACKUP_PASSCOMMAND="pass passwordname"
+BACKUP_PASSCOMMAND="echo superstrongpassword"
+#BACKUP_PASSCOMMAND="pass passwordname