diff --git a/Dockerfile b/Dockerfile index 2a9789a..6c3197b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -RUN apk add bash coreutils xxd +RUN apk add bash coreutils xxd restic WORKDIR /code COPY ./backup.sh . diff --git a/backup.sh b/backup.sh index 750e217..06bad21 100755 --- a/backup.sh +++ b/backup.sh @@ -38,7 +38,7 @@ debug-log () { fi } -while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do +while getopts 'a:cd:e:f:hi:l:m:o:p:qr:s:vw:' FLAG; do case $FLAG in a) COMPRESSION_ALGORITHM=$OPTARG ;; c) ENABLE_CHAT_MESSAGES=true ;; @@ -57,6 +57,7 @@ while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do echo "-l Compression level (default: 3)" echo "-m Maximum backups to keep, use -1 for unlimited (default: 128)" echo "-o Output directory" + echo "-r Restic repo name (if using restic)" echo "-p Prefix that shows in Minecraft chat (default: Backup)" echo "-q Suppress warnings" echo "-s Screen name, tmux session name, or hostname:port:password for RCON" @@ -68,6 +69,7 @@ while getopts 'a:cd:e:f:hi:l:m:o:p:qs:vw:' FLAG; do l) COMPRESSION_LEVEL=$OPTARG ;; m) MAX_BACKUPS=$OPTARG ;; o) BACKUP_DIRECTORY=$OPTARG ;; + r) RESTIC_REPO=$OPTARG ;; p) PREFIX=$OPTARG ;; q) SUPPRESS_WARNINGS=true ;; s) SCREEN_NAME=$OPTARG ;; @@ -190,26 +192,42 @@ rcon-command () { exec 3>&- } -if [[ $COMPRESSION_FILE_EXTENSION == "." ]]; then +if ! "$DEBUG"; then + QUIET="-q" +else + QUIET="" +fi + +if [[ "$COMPRESSION_FILE_EXTENSION" == "." ]]; then COMPRESSION_FILE_EXTENSION="" fi # Check for missing encouraged arguments if ! $SUPPRESS_WARNINGS; then - if [[ $SCREEN_NAME == "" ]]; then + if [[ "$SCREEN_NAME" == "" ]]; then log-warning "Minecraft screen/tmux/rcon location not specified (use -s)" fi fi # Check for required arguments MISSING_CONFIGURATION=false -if [[ $SERVER_WORLD == "" ]]; then +if [[ "$SERVER_WORLD" == "" ]]; then log-fatal "Server world not specified (use -i)" MISSING_CONFIGURATION=true fi -if [[ $BACKUP_DIRECTORY == "" ]]; then - log-fatal "Backup directory not specified (use -o)" +if [[ "$BACKUP_DIRECTORY" == "" ]] && [[ "$RESTIC_REPO" == "" ]]; then + log-fatal "Backup location not specified (use -o or -r)" MISSING_CONFIGURATION=true fi +if [[ "$RESTIC_REPO" != "" ]]; then + if [[ "$BACKUP_DIRECTORY" != "" ]]; then + log-fatal "Both output directory (-o) and restic repo (-r) specified but only one may be used at a time" + MISSING_CONFIGURATION=true + fi + if [[ $MAX_BACKUPS -ge 0 ]] && [[ $MAX_BACKUPS -lt 70 ]] && [[ $DELETE_METHOD == "thin" ]]; then + log-fatal "Thinning delete with restic requires at least 70 snapshots to be kept. If you need to keep fewer than 70, use sequential delete." + MISSING_CONFIGURATION=true + fi +fi if $MISSING_CONFIGURATION; then exit 1 @@ -257,9 +275,6 @@ message-players-color () { fi } -# Notify players of start -message-players "Starting backup..." "$ARCHIVE_FILE_NAME" - # Parse file timestamp to one readable by "date" parse-file-timestamp () { local DATE_STRING @@ -359,7 +374,7 @@ delete-thinning () { OLDEST_BACKUP_TIMESTAMP=$(parse-file-timestamp "${OLDEST_BACKUP_IN_BLOCK:0:19}") local BLOCK_COMMAND="$BLOCK_FUNCTION $OLDEST_BACKUP_TIMESTAMP" - if $BLOCK_COMMAND; then + if $BLOCK_COMMAND; then # Oldest backup in this block satisfies the condition for placement in the next block debug-log "$OLDEST_BACKUP_IN_BLOCK promoted to next block" else @@ -374,14 +389,41 @@ delete-thinning () { delete-sequentially } +delete-restic-sequential () { + if [ "$MAX_BACKUPS" -ge 0 ]; then + restic forget -r "$RESTIC_REPO" --keep-last "$MAX_BACKUPS" "$QUIET" + fi +} + +delete-restic-thinning () { + if [ "$MAX_BACKUPS" -ge 70 ]; then + # MAX_BACKUPS >= 70 + restic forget -r "$RESTIC_REPO" --keep-last 16 --keep-hourly 24 --keep-daily 30 --keep-weekly $((MAX_BACKUPS - 70)) "$QUIET" + else + # We have a check that MAX_BACKUPS is not 70 > MAX_BACKUPS >= 0, so we can assume here it is negative + # Negative means don't delete old snapshots + restic forget -r "$RESTIC_REPO" --keep-last 16 --keep-hourly 24 --keep-daily 30 --keep-weekly 9999999 "$QUIET" + fi +} + # Delete old backups delete-old-backups () { - case $DELETE_METHOD in - "sequential") delete-sequentially - ;; - "thin") delete-thinning - ;; - esac + if [[ "$BACKUP_DIRECTORY" != "" ]]; then + case $DELETE_METHOD in + "sequential") delete-sequentially + ;; + "thin") delete-thinning + ;; + esac + fi + if [[ "$RESTIC_REPO" != "" ]]; then + case $DELETE_METHOD in + "sequential") delete-restic-sequential + ;; + "thin") delete-restic-thinning + ;; + esac + fi } clean-up () { @@ -391,59 +433,97 @@ clean-up () { # Save the world execute-command "save-all" - # Notify players of completion - WORLD_SIZE_BYTES=$(du -b --max-depth=0 "$SERVER_WORLD" | awk '{print $1}') - ARCHIVE_SIZE_BYTES=$(du -b "$ARCHIVE_PATH" | awk '{print $1}') - ARCHIVE_SIZE=$(du -h "$ARCHIVE_PATH" | awk '{print $1}') - BACKUP_DIRECTORY_SIZE=$(du -h --max-depth=0 "$BACKUP_DIRECTORY" | awk '{print $1}') TIME_DELTA=$((END_TIME - START_TIME)) - # Check that archive size is not null and at least 200 Bytes - if [[ "$ARCHIVE_EXIT_CODE" == "0" && "$WORLD_SIZE_BYTES" -gt 0 && "$ARCHIVE_SIZE" != "" && "$ARCHIVE_SIZE_BYTES" -gt 200 ]]; then - COMPRESSION_PERCENT=$((ARCHIVE_SIZE_BYTES * 100 / WORLD_SIZE_BYTES)) - message-players-success "Backup complete!" "$TIME_DELTA s, $ARCHIVE_SIZE/$BACKUP_DIRECTORY_SIZE, $COMPRESSION_PERCENT%" - delete-old-backups - exit 0 - else - rm "$ARCHIVE_PATH" # Delete bad archive so we can't fill up with bad archives - message-players-error "Backup was not saved!" "Please notify an administrator" - exit 1 + if [[ "$BACKUP_DIRECTORY" != "" ]]; then + WORLD_SIZE_BYTES=$(du -b --max-depth=0 "$SERVER_WORLD" | awk '{print $1}') + ARCHIVE_SIZE_BYTES=$(du -b "$ARCHIVE_PATH" | awk '{print $1}') + ARCHIVE_SIZE=$(du -h "$ARCHIVE_PATH" | awk '{print $1}') + BACKUP_DIRECTORY_SIZE=$(du -h --max-depth=0 "$BACKUP_DIRECTORY" | awk '{print $1}') + + # Check that archive size is not null and at least 200 Bytes + if [[ "$ARCHIVE_EXIT_CODE" == "0" && "$WORLD_SIZE_BYTES" -gt 0 && "$ARCHIVE_SIZE" != "" && "$ARCHIVE_SIZE_BYTES" -gt 200 ]]; then + # Notify players of completion + COMPRESSION_PERCENT=$((ARCHIVE_SIZE_BYTES * 100 / WORLD_SIZE_BYTES)) + message-players-success "Backup complete!" "$TIME_DELTA s, $ARCHIVE_SIZE/$BACKUP_DIRECTORY_SIZE, $COMPRESSION_PERCENT%" + delete-old-backups + exit 0 + else + rm "$ARCHIVE_PATH" # Delete bad archive so we can't fill up with bad archives + message-players-error "Backup was not saved!" "Please notify an administrator" + exit 1 + fi + fi + + if [[ "$RESTIC_REPO" != "" ]]; then + if [[ "$ARCHIVE_EXIT_CODE" == "0" ]]; then + message-players-success "Backup complete!" "$TIME_DELTA s" + delete-old-backups + exit 0 + else + message-players-error "Backup was not saved!" "Please notify an administrator" + exit 1 + fi fi } -trap "clean-up" 2 +# Notify players of start +message-players "Starting backup..." "$ARCHIVE_FILE_NAME" -# Ensure backup directory exists -mkdir -p "$(dirname "$ARCHIVE_PATH")" +trap "clean-up" 2 # Disable world autosaving execute-command "save-off" # Backup world START_TIME=$(date +"%s") -case $COMPRESSION_ALGORITHM in - # No compression - "") tar -cf "$ARCHIVE_PATH" -C "$SERVER_WORLD" . - ;; - # With compression - *) tar -cf - -C "$SERVER_WORLD" . | $COMPRESSION_ALGORITHM -cv -"$COMPRESSION_LEVEL" - > "$ARCHIVE_PATH" 2>> /dev/null - ;; -esac -EXIT_CODES=("${PIPESTATUS[@]}") -# tar exit codes: http://www.gnu.org/software/tar/manual/html_section/Synopsis.html -# 0 = successful, 1 = some files differ, 2 = fatal -if [ "${EXIT_CODES[0]}" == "1" ]; then - log-warning "Some files may differ in the backup archive (file changed as read)" - TAR_EXIT_CODE="0" -else - TAR_EXIT_CODE="${EXIT_CODES[0]}" +if [[ "$BACKUP_DIRECTORY" != "" ]]; then + # Ensure backup directory exists + mkdir -p "$(dirname "$ARCHIVE_PATH")" + + case $COMPRESSION_ALGORITHM in + # No compression + "") tar -cf "$ARCHIVE_PATH" -C "$SERVER_WORLD" . + ;; + # With compression + *) tar -cf - -C "$SERVER_WORLD" . | $COMPRESSION_ALGORITHM -cv -"$COMPRESSION_LEVEL" - > "$ARCHIVE_PATH" 2>> /dev/null + ;; + esac + EXIT_CODES=("${PIPESTATUS[@]}") + + # tar exit codes: http://www.gnu.org/software/tar/manual/html_section/Synopsis.html + # 0 = successful, 1 = some files differ, 2 = fatal + if [ "${EXIT_CODES[0]}" == "1" ]; then + log-warning "Some files may differ in the backup archive (file changed as read)" + TAR_EXIT_CODE="0" + else + TAR_EXIT_CODE="${EXIT_CODES[0]}" + fi + + ARCHIVE_EXIT_CODE="$(exit-code "$TAR_EXIT_CODE" "${EXIT_CODES[1]}")" + if [ "$ARCHIVE_EXIT_CODE" -ne 0 ]; then + log-fatal "Archive command exited with nonzero exit code $ARCHIVE_EXIT_CODE" + fi fi -ARCHIVE_EXIT_CODE="$(exit-code "$TAR_EXIT_CODE" "${EXIT_CODES[1]}")" -if [ "$ARCHIVE_EXIT_CODE" -ne 0 ]; then - log-fatal "Archive command exited with nonzero exit code $ARCHIVE_EXIT_CODE" +if [[ "$RESTIC_REPO" != "" ]]; then + RESTIC_TIMESTAMP="${TIMESTAMP:0:10} ${TIMESTAMP:11:2}:${TIMESTAMP:14:2}:${TIMESTAMP:17:2}" + restic backup -r "$RESTIC_REPO" "$SERVER_WORLD" --time "$RESTIC_TIMESTAMP" "$QUIET" + ARCHIVE_EXIT_CODE=$? + if [ "$ARCHIVE_EXIT_CODE" -eq 3 ]; then + log-warning "Incomplete snapshot taken (some files could not be read)" + ARCHIVE_EXIT_CODE="0" + else + if [ "$ARCHIVE_EXIT_CODE" -ne 0 ]; then + # According to the restic docs, exit code is either 0, 1, or 3 + # Exit code 1 means fatal + # See: https://restic.readthedocs.io/en/latest/040_backup.html + log-fatal "No restic snapshot created (exit code $ARCHIVE_EXIT_CODE)" + fi + fi fi + sync END_TIME=$(date +"%s") diff --git a/test/test.sh b/test/test.sh index dba1b68..21179fa 100755 --- a/test/test.sh +++ b/test/test.sh @@ -7,13 +7,16 @@ TEST_TMP="$TEST_DIR/tmp" SCREEN_TMP="tmp-screen" RCON_PORT="8088" RCON_PASSWORD="supersecret" +export RESTIC_PASSWORD="restic-pass-secret" setUp () { + chmod -R 755 "$TEST_TMP" rm -rf "$TEST_TMP" mkdir -p "$TEST_TMP/server/world" mkdir -p "$TEST_TMP/backups" echo "file1" > "$TEST_TMP/server/world/file1.txt" echo "file2" > "$TEST_TMP/server/world/file2.txt" echo "file3" > "$TEST_TMP/server/world/file3.txt" + restic init -r "$TEST_TMP/backups-restic" -q screen -dmS "$SCREEN_TMP" bash while ! screen -S "$SCREEN_TMP" -Q "select" . &>/dev/null; do @@ -42,6 +45,9 @@ tearDown () { } assert-equals-directory () { + if ! [ -e "$1" ]; then + fail "File not found: $1" + fi if [ -d "$1" ]; then for FILE in "$1"/*; do assert-equals-directory "$FILE" "$2/${FILE##$1}" @@ -65,8 +71,100 @@ check-backup () { check-backup-full-paths "$TEST_TMP/backups/$BACKUP_ARCHIVE" "$TEST_TMP/server/world" } +check-latest-backup-restic () { + WORLD_DIR="$TEST_TMP/server/world" + restic restore latest -r "$TEST_TMP/backups-restic" --target "$TEST_TMP/restored" -q + assert-equals-directory "$WORLD_DIR" "$TEST_TMP/restored/$WORLD_DIR" + rm -rf "$TEST_TMP/restored" +} + # Tests +test-restic-incomplete-snapshot () { + chmod 000 "$TEST_TMP/server/world/file1.txt" + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")" + OUTPUT="$(./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP")" + assertContains "$OUTPUT" "Incomplete snapshot taken" +} + +test-restic-no-snapshot () { + rm -rf "$TEST_TMP/server" + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")" + OUTPUT="$(./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP")" + EXIT_CODE="$?" + assertNotEquals 0 "$EXIT_CODE" + assertContains "$OUTPUT" "No restic snapshot created" +} + +test-restic-thinning-too-few () { + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")" + OUTPUT="$(./backup.sh -m 10 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" 2>&1)" + EXIT_CODE="$?" + assertNotEquals 0 "$EXIT_CODE" + assertContains "$OUTPUT" "Thinning delete with restic requires at least 70 snapshots to be kept." +} + +test-restic-thinning-delete-long () { + for i in $(seq 0 99); do + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i day")" + ./backup.sh -m -1 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" + done + EXPECTED_TIMESTAMPS=( + # Weekly + "2021-01-03 00:00:00" + "2021-01-10 00:00:00" + "2021-01-17 00:00:00" + "2021-01-24 00:00:00" + "2021-01-31 00:00:00" + + # Daily (30) + "2021-03-13 00:00:00" + "2021-03-14 00:00:00" + "2021-03-15 00:00:00" + "2021-03-16 00:00:00" + "2021-03-17 00:00:00" + "2021-03-18 00:00:00" + + # Hourly (24) + "2021-03-19 00:00:00" + "2021-03-20 00:00:00" + "2021-03-21 00:00:00" + "2021-03-22 00:00:00" + "2021-03-23 00:00:00" + "2021-03-24 00:00:00" + "2021-03-25 00:00:00" + "2021-03-26 00:00:00" + + # Sub-hourly (16) + "2021-03-26 00:00:00" + "2021-03-27 00:00:00" + "2021-03-28 00:00:00" + "2021-03-29 00:00:00" + "2021-03-30 00:00:00" + "2021-03-31 00:00:00" + "2021-04-01 00:00:00" + "2021-04-02 00:00:00" + "2021-04-03 00:00:00" + "2021-04-04 00:00:00" + "2021-04-05 00:00:00" + "2021-04-06 00:00:00" + "2021-04-07 00:00:00" + "2021-04-08 00:00:00" + "2021-04-09 00:00:00" + "2021-04-10 00:00:00" + ) + SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")" + for TIMESTAMP in "${EXPECTED_TIMESTAMPS[@]}"; do + assertContains "$SNAPSHOTS" "$TIMESTAMP" + done +} + +test-restic-defaults () { + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")" + ./backup.sh -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" + check-latest-backup-restic +} + test-backup-defaults () { TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01")" ./backup.sh -i "$TEST_TMP/server/world" -o "$TEST_TMP/backups" -s "$SCREEN_TMP" -f "$TIMESTAMP" @@ -123,7 +221,14 @@ test-missing-options () { assertEquals 1 "$EXIT_CODE" assertContains "$OUTPUT" "Minecraft screen/tmux/rcon location not specified (use -s)" assertContains "$OUTPUT" "Server world not specified" - assertContains "$OUTPUT" "Backup directory not specified" + assertContains "$OUTPUT" "Backup location not specified" +} + +test-restic-and-output-options () { + OUTPUT="$(./backup.sh -c -i "$TEST_TMP/server/world" -o "$TEST_TMP/backups" -s "$SCREEN_TMP" -f "$TIMESTAMP" -r "$TEST_TMP/backups-restic" 2>&1)" + EXIT_CODE="$?" + assertEquals 1 "$EXIT_CODE" + assertContains "$OUTPUT" "Both output directory (-o) and restic repo (-r) specified but only one may be used at a time" } test-missing-options-suppress-warnings () { @@ -242,6 +347,24 @@ test-sequential-delete () { assertEquals 10 "$(find "$TEST_TMP/backups" -type f | wc -l)" } +test-restic-sequential-delete () { + for i in $(seq 0 20); do + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")" + ./backup.sh -d "sequential" -m 10 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" + done + assertEquals 10 "$(restic list snapshots -r "$TEST_TMP/backups-restic" | wc -l)" + check-latest-backup-restic + SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")" + for i in $(seq 11 20); do + TIMESTAMP="$(date "+%F %H:%M:%S" --date="2021-01-01 +$i hour")" + assertContains "$SNAPSHOTS" "$TIMESTAMP" + done + for i in $(seq 0 10); do + TIMESTAMP="$(date "+%F %H:%M:%S" --date="2021-01-01 +$i hour")" + assertNotContains "$SNAPSHOTS" "$TIMESTAMP" + done +} + test-thinning-delete () { for i in $(seq 0 99); do TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")" @@ -292,6 +415,60 @@ test-thinning-delete () { done } +test-restic-thinning-delete () { + for i in $(seq 0 99); do + TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i hour")" + ./backup.sh -m 70 -i "$TEST_TMP/server/world" -r "$TEST_TMP/backups-restic" -s "$SCREEN_TMP" -f "$TIMESTAMP" + done + EXPECTED_TIMESTAMPS=( + # Weekly + + # Daily (30) + "2021-01-01 23:00:00" + "2021-01-02 23:00:00" + "2021-01-03 23:00:00" + + # Hourly (24) + "2021-01-04 04:00:00" + "2021-01-04 05:00:00" + "2021-01-04 06:00:00" + "2021-01-04 07:00:00" + "2021-01-04 08:00:00" + "2021-01-04 09:00:00" + "2021-01-04 10:00:00" + "2021-01-04 11:00:00" + + # Sub-hourly (16) + "2021-01-04 12:00:00" + "2021-01-04 13:00:00" + "2021-01-04 14:00:00" + "2021-01-04 15:00:00" + "2021-01-04 16:00:00" + "2021-01-04 17:00:00" + "2021-01-04 18:00:00" + "2021-01-04 19:00:00" + "2021-01-04 20:00:00" + "2021-01-04 21:00:00" + "2021-01-04 22:00:00" + "2021-01-04 23:00:00" + "2021-01-05 00:00:00" + "2021-01-05 01:00:00" + "2021-01-05 02:00:00" + "2021-01-05 03:00:00" + ) + SNAPSHOTS="$(restic snapshots -r "$TEST_TMP/backups-restic")" + for TIMESTAMP in "${EXPECTED_TIMESTAMPS[@]}"; do + assertContains "$SNAPSHOTS" "$TIMESTAMP" + done + UNEXPECTED_TIMESTAMPS=( + "2021-01-01 00:00:00" + "2021-01-02 22:00:00" + ) + for TIMESTAMP in "${UNEXPECTED_TIMESTAMPS[@]}"; do + assertNotContains "$SNAPSHOTS" "$TIMESTAMP" + done +} + test-thinning-delete-long () { for i in $(seq 0 99); do TIMESTAMP="$(date +%F_%H-%M-%S --date="2021-01-01 +$i day")"