Add restic support

Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine
 
-RUN apk add bash coreutils xxd
+RUN apk add bash coreutils xxd restic
 
 WORKDIR /code
 COPY ./backup.sh .

backup.sh

@@ -38,7 +38,7 @@ debug-log () {
   fi
 }
 
-while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do
+while getopts 'a:cd:e:f:hi:l:m:o:p:qr:s:vw:' FLAG; do
   case $FLAG in
     a) COMPRESSION_ALGORITHM=$OPTARG ;;
     c) ENABLE_CHAT_MESSAGES=true ;;
@@ -57,6 +57,7 @@ while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do
        echo "-l    Compression level (default: 3)"
        echo "-m    Maximum backups to keep, use -1 for unlimited (default: 128)"
        echo "-o    Output directory"
+       echo "-r    Restic repo name (if using restic)"
        echo "-p    Prefix that shows in Minecraft chat (default: Backup)"
        echo "-q    Suppress warnings"
        echo "-s    Screen name, tmux session name, or hostname:port:password for RCON"
@@ -68,6 +69,7 @@ while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do
     l) COMPRESSION_LEVEL=$OPTARG ;;
     m) MAX_BACKUPS=$OPTARG ;;
     o) BACKUP_DIRECTORY=$OPTARG ;;
+    r) RESTIC_REPO=$OPTARG ;;
     p) PREFIX=$OPTARG ;;
     q) SUPPRESS_WARNINGS=true ;;
     s) SCREEN_NAME=$OPTARG ;;
@@ -190,26 +192,42 @@ rcon-command () {
   exec 3>&-
 }
 
-if [[ $COMPRESSION_FILE_EXTENSION == "." ]]; then
+if ! "$DEBUG"; then
+  QUIET="-q"
+else
+  QUIET=""
+fi
+
+if [[ "$COMPRESSION_FILE_EXTENSION" == "." ]]; then
   COMPRESSION_FILE_EXTENSION=""
 fi
 
 # Check for missing encouraged arguments
 if ! $SUPPRESS_WARNINGS; then
-  if [[ $SCREEN_NAME == "" ]]; then
+  if [[ "$SCREEN_NAME" == "" ]]; then
     log-warning "Minecraft screen/tmux/rcon location not specified (use -s)"
   fi
 fi
 # Check for required arguments
 MISSING_CONFIGURATION=false
-if [[ $SERVER_WORLD == "" ]]; then
+if [[ "$SERVER_WORLD" == "" ]]; then
   log-fatal "Server world not specified (use -i)"
   MISSING_CONFIGURATION=true
 fi
-if [[ $BACKUP_DIRECTORY == "" ]]; then
+if [[ "$BACKUP_DIRECTORY" == "" ]] && [[ "$RESTIC_REPO" == "" ]]; then
-  log-fatal "Backup directory not specified (use -o)"
+  log-fatal "Backup location not specified (use -o or -r)"
   MISSING_CONFIGURATION=true
 fi
+if [[ "$RESTIC_REPO" != "" ]]; then
+  if [[ "$BACKUP_DIRECTORY" != "" ]]; then
+    log-fatal "Both output directory (-o) and restic repo (-r) specified but only one may be used at a time"
+    MISSING_CONFIGURATION=true
+  fi
+  if [[ $MAX_BACKUPS -ge 0 ]] && [[ $MAX_BACKUPS -lt 70 ]] && [[ $DELETE_METHOD == "thin" ]]; then
+    log-fatal "Thinning delete with restic requires at least 70 snapshots to be kept. If you need to keep fewer than 70, use sequential delete."
+    MISSING_CONFIGURATION=true
+  fi
+fi
 
 if $MISSING_CONFIGURATION; then
   exit 1
@@ -257,9 +275,6 @@ message-players-color () {
   fi
 }
 
-# Notify players of start
-message-players "Starting backup..." "$ARCHIVE_FILE_NAME"
-
 # Parse file timestamp to one readable by "date" 
 parse-file-timestamp () {
   local DATE_STRING
@@ -374,14 +389,41 @@ delete-thinning () {
   delete-sequentially
 }
 
+delete-restic-sequential () {
+  if [ "$MAX_BACKUPS" -ge 0 ]; then
+    restic forget -r "$RESTIC_REPO" --keep-last "$MAX_BACKUPS" "$QUIET"
+  fi
+}
+
+delete-restic-thinning () {
+  if [ "$MAX_BACKUPS" -ge 70 ]; then 
+    # MAX_BACKUPS >= 70
+    restic forget -r "$RESTIC_REPO" --keep-last 16 --keep-hourly 24 --keep-daily 30 --keep-weekly $((MAX_BACKUPS - 70)) "$QUIET"
+  else 
+    # We have a check that MAX_BACKUPS is not 70 > MAX_BACKUPS >= 0, so we can assume here it is negative
+    # Negative means don't delete old snapshots
+    restic forget -r "$RESTIC_REPO" --keep-last 16 --keep-hourly 24 --keep-daily 30 --keep-weekly 9999999 "$QUIET"
+  fi
+}
+
 # Delete old backups
 delete-old-backups () {
+  if [[ "$BACKUP_DIRECTORY" != "" ]]; then
     case $DELETE_METHOD in
       "sequential") delete-sequentially
         ;;
       "thin") delete-thinning
         ;;
     esac
+  fi
+  if [[ "$RESTIC_REPO" != "" ]]; then
+    case $DELETE_METHOD in
+      "sequential") delete-restic-sequential
+        ;;
+     "thin") delete-restic-thinning
+        ;;
+    esac
+  fi
 }
 
 clean-up () {
@@ -391,15 +433,17 @@ clean-up () {
   # Save the world
   execute-command "save-all"
 
-  # Notify players of completion
+  TIME_DELTA=$((END_TIME - START_TIME))
+
+  if [[ "$BACKUP_DIRECTORY" != "" ]]; then
     WORLD_SIZE_BYTES=$(du -b --max-depth=0 "$SERVER_WORLD" | awk '{print $1}')
     ARCHIVE_SIZE_BYTES=$(du -b "$ARCHIVE_PATH" | awk '{print $1}')
     ARCHIVE_SIZE=$(du -h "$ARCHIVE_PATH" | awk '{print $1}')
     BACKUP_DIRECTORY_SIZE=$(du -h --max-depth=0 "$BACKUP_DIRECTORY" | awk '{print $1}')
-  TIME_DELTA=$((END_TIME - START_TIME))
 
     # Check that archive size is not null and at least 200 Bytes
     if [[ "$ARCHIVE_EXIT_CODE" == "0" && "$WORLD_SIZE_BYTES" -gt 0 && "$ARCHIVE_SIZE" != "" && "$ARCHIVE_SIZE_BYTES" -gt 200 ]]; then
+      # Notify players of completion
       COMPRESSION_PERCENT=$((ARCHIVE_SIZE_BYTES * 100 / WORLD_SIZE_BYTES))
       message-players-success "Backup complete!" "$TIME_DELTA s, $ARCHIVE_SIZE/$BACKUP_DIRECTORY_SIZE, $COMPRESSION_PERCENT%"
       delete-old-backups
@@ -409,18 +453,35 @@ clean-up () {
       message-players-error "Backup was not saved!" "Please notify an administrator"
       exit 1
     fi
+  fi
+
+  if [[ "$RESTIC_REPO" != "" ]]; then
+    if [[ "$ARCHIVE_EXIT_CODE" == "0" ]]; then
+      message-players-success "Backup complete!" "$TIME_DELTA s"
+      delete-old-backups
+      exit 0
+    else
+      message-players-error "Backup was not saved!" "Please notify an administrator"
+      exit 1
+    fi
+  fi
 }
 
-trap "clean-up" 2
+# Notify players of start
+message-players "Starting backup..." "$ARCHIVE_FILE_NAME"
 
-# Ensure backup directory exists
+trap "clean-up" 2
-mkdir -p "$(dirname "$ARCHIVE_PATH")"
 
 # Disable world autosaving
 execute-command "save-off"
 
 # Backup world
 START_TIME=$(date +"%s")
+
+if [[ "$BACKUP_DIRECTORY" != "" ]]; then
+  # Ensure backup directory exists
+  mkdir -p "$(dirname "$ARCHIVE_PATH")"
+
   case $COMPRESSION_ALGORITHM in
     # No compression
     "") tar -cf "$ARCHIVE_PATH" -C "$SERVER_WORLD" .
@@ -444,6 +505,25 @@ ARCHIVE_EXIT_CODE="$(exit-code "$TAR_EXIT_CODE" "${EXIT_CODES[1]}")"
   if [ "$ARCHIVE_EXIT_CODE" -ne 0 ]; then
     log-fatal "Archive command exited with nonzero exit code $ARCHIVE_EXIT_CODE"
   fi
+fi
+
+if [[ "$RESTIC_REPO" != "" ]]; then
+  RESTIC_TIMESTAMP="${TIMESTAMP:0:10} ${TIMESTAMP:11:2}:${TIMESTAMP:14:2}:${TIMESTAMP:17:2}"
+  restic backup -r "$RESTIC_REPO" "$SERVER_WORLD" --time "$RESTIC_TIMESTAMP" "$QUIET"
+  ARCHIVE_EXIT_CODE=$?
+  if [ "$ARCHIVE_EXIT_CODE" -eq 3 ]; then
+    log-warning "Incomplete snapshot taken (some files could not be read)"
+    ARCHIVE_EXIT_CODE="0"
+  else 
+    if [ "$ARCHIVE_EXIT_CODE" -ne 0 ]; then
+      # According to the restic docs, exit code is either 0, 1, or 3
+      # Exit code 1 means fatal
+      # See: https://restic.readthedocs.io/en/latest/040_backup.html
+      log-fatal "No restic snapshot created (exit code $ARCHIVE_EXIT_CODE)"
+    fi
+  fi
+fi
+
 sync
 END_TIME=$(date +"%s")
 

test/test.sh

@@ -7,13 +7,16 @@ TEST_TMP="$TEST_DIR/tmp"
 SCREEN_TMP="tmp-screen"
 RCON_PORT="8088"
 RCON_PASSWORD="supersecret"
+export RESTIC_PASSWORD="restic-pass-secret"
 setUp () {
+  chmod -R 755 "$TEST_TMP"
   rm -rf "$TEST_TMP"
   mkdir -p "$TEST_TMP/server/world"
   mkdir -p "$TEST_TMP/backups"
   echo "file1" > "$TEST_TMP/server/world/file1.txt"
   echo "file2" > "$TEST_TMP/server/world/file2.txt"
   echo "file3" > "$TEST_TMP/server/world/file3.txt"
+  restic init -r "$TEST_TMP/backups-restic" -q
 
   screen -dmS "$SCREEN_TMP" bash
   while ! screen -S "$SCREEN_TMP" -Q "select" . &>/dev/null; do
@@ -42,6 +45,9 @@ tearDown () {
 }
 
 assert-equals-directory () {
+  if ! [ -e "$1" ]; then
+    fail "File not found: $1"
+  fi
   if [ -d "$1" ]; then
     for FILE in "$1"/*; do
       assert-equals-directory "$FILE" "$2/${FILE##$1}"
@@ -65,8 +71,100 @@ check-backup () {
   check-backup-full-paths "$TEST_TMP/backups/$BACKUP_ARCHIVE" "$TEST_TMP/server/world"
 }
 
+check-latest-backup-restic () {
+  WORLD_DIR="$TEST_TMP/server/world"
+  restic restore latest -r "$TEST_TMP/backups-restic" --target "$TEST_TMP/restored" -q
+  assert-equals-directory "$WORLD_DIR" "$TEST_TMP/restored/$WORLD_DIR"
+  rm -rf "$TEST_TMP/restored"
+}
+
 # Tests
 
+test-restic-incomplete-snapshot () {
+  chmod 000 "$TEST_TMP/server/world/file1.txt"
+  TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")"
+  OUTPUT="$(./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP")"
+  assertContains "$OUTPUT" "Incomplete snapshot taken"
+}
+
+test-restic-no-snapshot () {
+  rm -rf "$TEST_TMP/server"
+  TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")"
+  OUTPUT="$(./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP")"
+  EXIT_CODE="$?"
+  assertNotEquals 0 "$EXIT_CODE"
+  assertContains "$OUTPUT" "No restic snapshot created"
+}
+
+test-restic-thinning-too-few () {
+  TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")"
+  OUTPUT="$(./backup.sh -m 10 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" 2>&1)"
+  EXIT_CODE="$?"
+  assertNotEquals 0 "$EXIT_CODE"
+  assertContains "$OUTPUT" "Thinning delete with restic requires at least 70 snapshots to be kept."
+}
+
+test-restic-thinning-delete-long () {
+  for i in $(seq 0 99); do
+    TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i day")"
+    ./backup.sh -m -1 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP"
+  done
+  EXPECTED_TIMESTAMPS=(
+    # Weekly
+    "2021-01-03 00:00:00"
+    "2021-01-10 00:00:00"
+    "2021-01-17 00:00:00"
+    "2021-01-24 00:00:00"
+    "2021-01-31 00:00:00"
+
+    # Daily (30)
+    "2021-03-13 00:00:00"
+    "2021-03-14 00:00:00"
+    "2021-03-15 00:00:00"
+    "2021-03-16 00:00:00"
+    "2021-03-17 00:00:00"
+    "2021-03-18 00:00:00"
+
+    # Hourly (24)
+    "2021-03-19 00:00:00"
+    "2021-03-20 00:00:00"
+    "2021-03-21 00:00:00"
+    "2021-03-22 00:00:00"
+    "2021-03-23 00:00:00"
+    "2021-03-24 00:00:00"
+    "2021-03-25 00:00:00"
+    "2021-03-26 00:00:00"
+
+    # Sub-hourly (16)
+    "2021-03-26 00:00:00"
+    "2021-03-27 00:00:00"
+    "2021-03-28 00:00:00"
+    "2021-03-29 00:00:00"
+    "2021-03-30 00:00:00"
+    "2021-03-31 00:00:00"
+    "2021-04-01 00:00:00"
+    "2021-04-02 00:00:00"
+    "2021-04-03 00:00:00"
+    "2021-04-04 00:00:00"
+    "2021-04-05 00:00:00"
+    "2021-04-06 00:00:00"
+    "2021-04-07 00:00:00"
+    "2021-04-08 00:00:00"
+    "2021-04-09 00:00:00"
+    "2021-04-10 00:00:00"
+  )
+  SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")"
+  for TIMESTAMP in "${EXPECTED_TIMESTAMPS[@]}"; do
+    assertContains "$SNAPSHOTS" "$TIMESTAMP" 
+  done
+}
+
+test-restic-defaults () {
+  TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")"
+  ./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP"
+  check-latest-backup-restic
+}
+
 test-backup-defaults () {
   TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")"
   ./backup.sh -i "$TEST_TMP/server/world" -o "$TEST_TMP/backups" -s "$SCREEN_TMP" -f "$TIMESTAMP"
@@ -123,7 +221,14 @@ test-missing-options () {
   assertEquals 1 "$EXIT_CODE"
   assertContains "$OUTPUT" "Minecraft screen/tmux/rcon location not specified (use -s)"
   assertContains "$OUTPUT" "Server world not specified"
-  assertContains "$OUTPUT" "Backup directory not specified"
+  assertContains "$OUTPUT" "Backup location not specified"
+}
+
+test-restic-and-output-options () {
+  OUTPUT="$(./backup.sh -c -i "$TEST_TMP/server/world" -o "$TEST_TMP/backups" -s "$SCREEN_TMP" -f "$TIMESTAMP" -r "$TEST_TMP/backups-restic" 2>&1)"
+  EXIT_CODE="$?"
+  assertEquals 1 "$EXIT_CODE"
+  assertContains "$OUTPUT" "Both output directory (-o) and restic repo (-r) specified but only one may be used at a time"
 }
 
 test-missing-options-suppress-warnings () {
@@ -242,6 +347,24 @@ test-sequential-delete () {
   assertEquals 10 "$(find "$TEST_TMP/backups" -type f | wc -l)" 
 }
 
+test-restic-sequential-delete () {
+  for i in $(seq 0 20); do
+    TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")"
+    ./backup.sh -d "sequential" -m 10 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP"
+  done
+  assertEquals 10 "$(restic list snapshots -r "$TEST_TMP/backups-restic" | wc -l)"
+  check-latest-backup-restic
+  SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")"
+  for i in $(seq 11 20); do
+    TIMESTAMP="$(date "+%F %H:%M:%S" --date="2021-01-01 +$i hour")"
+    assertContains "$SNAPSHOTS" "$TIMESTAMP" 
+  done
+  for i in $(seq 0 10); do
+    TIMESTAMP="$(date "+%F %H:%M:%S" --date="2021-01-01 +$i hour")"
+    assertNotContains "$SNAPSHOTS" "$TIMESTAMP" 
+  done
+}
+
 test-thinning-delete () {
   for i in $(seq 0 99); do
     TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")"
@@ -292,6 +415,60 @@ test-thinning-delete () {
   done
 }
 
+test-restic-thinning-delete () {
+  for i in $(seq 0 99); do
+    TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")"
+    ./backup.sh -m 70 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP"
+  done
+  EXPECTED_TIMESTAMPS=(
+    # Weekly
+
+    # Daily (30)
+    "2021-01-01 23:00:00"
+    "2021-01-02 23:00:00"
+    "2021-01-03 23:00:00"
+
+    # Hourly (24)
+    "2021-01-04 04:00:00"
+    "2021-01-04 05:00:00"
+    "2021-01-04 06:00:00"
+    "2021-01-04 07:00:00"
+    "2021-01-04 08:00:00"
+    "2021-01-04 09:00:00"
+    "2021-01-04 10:00:00"
+    "2021-01-04 11:00:00"
+
+    # Sub-hourly (16)
+    "2021-01-04 12:00:00"
+    "2021-01-04 13:00:00"
+    "2021-01-04 14:00:00"
+    "2021-01-04 15:00:00"
+    "2021-01-04 16:00:00"
+    "2021-01-04 17:00:00"
+    "2021-01-04 18:00:00"
+    "2021-01-04 19:00:00"
+    "2021-01-04 20:00:00"
+    "2021-01-04 21:00:00"
+    "2021-01-04 22:00:00"
+    "2021-01-04 23:00:00"
+    "2021-01-05 00:00:00"
+    "2021-01-05 01:00:00"
+    "2021-01-05 02:00:00"
+    "2021-01-05 03:00:00"
+  )
+  SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")"
+  for TIMESTAMP in "${EXPECTED_TIMESTAMPS[@]}"; do
+    assertContains "$SNAPSHOTS" "$TIMESTAMP" 
+  done
+  UNEXPECTED_TIMESTAMPS=(
+    "2021-01-01 00:00:00"
+    "2021-01-02 22:00:00"
+  )
+  for TIMESTAMP in "${UNEXPECTED_TIMESTAMPS[@]}"; do
+    assertNotContains "$SNAPSHOTS" "$TIMESTAMP" 
+  done
+}
+
 test-thinning-delete-long () {
   for i in $(seq 0 99); do
     TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i day")"