From 851c2a9718ff19be7e883031dbe4df6c35215c40 Mon Sep 17 00:00:00 2001 From: Nicolas Chan Date: Thu, 10 Aug 2017 22:18:04 -0700 Subject: [PATCH] Command line flags, better thinning --- backup.sh | 215 +++++++++++++++++++++++++++++++++++++++++------------- test.sh | 18 +++++ 2 files changed, 184 insertions(+), 49 deletions(-) mode change 100644 => 100755 backup.sh create mode 100755 test.sh diff --git a/backup.sh b/backup.sh old mode 100644 new mode 100755 index 6af8001..eb69a84 --- a/backup.sh +++ b/backup.sh @@ -7,22 +7,87 @@ # For Minecraft servers running in a GNU screen. # For most convenience, run automatically with cron. -# Configuration -SCREEN_NAME="PrivateSurvival" # Name of the GNU Screen your Minecraft server is running in -SERVER_DIRECTORY="/home/server/MinecraftServers/PrivateSurvival" # Server directory, NOT the world; world is SERVER_DIRECTORY/world -BACKUP_DIRECTORY="/media/server/ExternalStorage/Backups/PrivateSurvivalBackups" # Directory to save backups in -NUMBER_OF_BACKUPS_TO_KEEP=512 # -1 indicates unlimited +# Default Configuration +SCREEN_NAME="" # Name of the GNU Screen your Minecraft server is running in +SERVER_WORLD="" # Server world directory +BACKUP_DIRECTORY="" # Directory to save backups in +MAX_BACKUPS=128 # -1 indicates unlimited DELETE_METHOD="thin" # Choices: thin, sequential, none; sequential: delete oldest; thin: keep last 24 hourly, last 30 daily, and monthly (use with 1 hr cron interval) -COMPRESSION_ALGORITHM="zstd" # Leave empty for no compression -COMPRESSION_FILE_EXTENSION=".zst" # Leave empty for no compression; Precede with a . (for example: ".gz") +COMPRESSION_ALGORITHM="gzip" # Leave empty for no compression +COMPRESSION_FILE_EXTENSION=".gz" # Leave empty for no compression; Precede with a . (for example: ".gz") COMPRESSION_LEVEL=3 # Passed to the compression algorithm -ENABLE_CHAT_MESSAGES=true # Tell players in Minecraft chat about backup status +ENABLE_CHAT_MESSAGES=false # Tell players in Minecraft chat about backup status PREFIX="Backup" # Shows in the chat message -DEBUG=true # Enable debug messages +DEBUG=false # Enable debug messages +SUPPRESS_WARNINGS=false # Suppress warnings # Other Variables (do not modify) DATE_FORMAT="%F_%H-%M-%S" TIMESTAMP=$(date +$DATE_FORMAT) + +while getopts 'a:cd:e:f:hi:l:m:o:p:qs:v' FLAG; do + case $FLAG in + a) COMPRESSION_ALGORITHM=$OPTARG ;; + c) ENABLE_CHAT_MESSAGES=true ;; + d) DELETE_METHOD=$OPTARG ;; + e) COMPRESSION_FILE_EXTENSION=".$OPTARG" ;; + f) TIMESTAMP=$OPTARG ;; + h) echo "Minecraft Backup (by Nicolas Chan)" + echo "-a Compression algorithm (default: gzip)" + echo "-c Enable chat messages" + echo "-d Delete method: thin (default), sequential, none" + echo "-e Compression file extension, exclude leading \".\" (default: gz)" + echo "-f Output file name (default is the timestamp)" + echo "-h Shows this help text" + echo "-i Input directory (path to world folder)" + echo "-l Compression level (default: 3)" + echo "-m Maximum backups to keep, use -1 for unlimited (default: 128)" + echo "-o Output directory" + echo "-p Prefix that shows in Minecraft chat (default: Backup)" + echo "-q Suppress warnings" + echo "-s Minecraft server screen name" + echo "-v Verbose mode" + exit 0 + ;; + i) SERVER_WORLD=$OPTARG ;; + l) COMPRESSION_LEVEL=$OPTARG ;; + m) MAX_BACKUPS=$OPTARG ;; + o) BACKUP_DIRECTORY=$OPTARG ;; + p) PREFIX=$OPTARG ;; + q) SUPPRESS_WARNINGS=true ;; + s) SCREEN_NAME=$OPTARG ;; + v) DEBUG=true ;; + esac +done + +log-fatal () { + echo -e "\033[0;31mFATAL:\033[0m $*" +} +log-warning () { + echo -e "\033[0;33mWARNING:\033[0m $*" +} + +# Check for missing encouraged arguments +if ! $SUPPRESS_WARNINGS; then + if [[ $SCREEN_NAME == "" ]]; then + log-warning "Minecraft screen name not specified (use -s)" + fi +fi +# Check for required arguments +MISSING_CONFIGURATION=false +if [[ $SERVER_WORLD == "" ]]; then + log-fatal "Server world not specified (use -i)" + MISSING_CONFIGURATION=true +fi +if [[ $BACKUP_DIRECTORY == "" ]]; then + log-fatal "Backup directory not specified (use -o)" + MISSING_CONFIGURATION=true +fi + +if $MISSING_CONFIGURATION; then + exit 0 +fi + ARCHIVE_FILE_NAME=$TIMESTAMP.tar$COMPRESSION_FILE_EXTENSION ARCHIVE_PATH=$BACKUP_DIRECTORY/$ARCHIVE_FILE_NAME @@ -34,7 +99,9 @@ message-players () { } execute-command () { local COMMAND=$1 - screen -S $SCREEN_NAME -p 0 -X stuff "$COMMAND$(printf \\r)" + if [[ $SCREEN_NAME != "" ]]; then + screen -S $SCREEN_NAME -p 0 -X stuff "$COMMAND$(printf \\r)" + fi } message-players-error () { local MESSAGE=$1 @@ -50,7 +117,9 @@ message-players-color () { local MESSAGE=$1 local HOVER_MESSAGE=$2 local COLOR=$3 - echo "$MESSAGE ($HOVER_MESSAGE)" + if $DEBUG; then + echo "$MESSAGE ($HOVER_MESSAGE)" + fi if $ENABLE_CHAT_MESSAGES; then execute-command "tellraw @a [\"\",{\"text\":\"[$PREFIX] \",\"color\":\"gray\",\"italic\":true},{\"text\":\"$MESSAGE\",\"color\":\"$COLOR\",\"italic\":true,\"hoverEvent\":{\"action\":\"show_text\",\"value\":{\"text\":\"\",\"extra\":[{\"text\":\"$HOVER_MESSAGE\"}]}}}]" fi @@ -59,53 +128,101 @@ message-players-color () { # Notify players of start message-players "Starting backup..." "$ARCHIVE_FILE_NAME" +# Parse file timestamp to one readable by "date" +parse-file-timestamp () { + local DATE_STRING=$(echo $1 | awk -F_ '{gsub(/-/,":",$2); print $1" "$2}') + echo $DATE_STRING +} + +# Delete a backup +delete-backup () { + local BACKUP=$1 + rm $BACKUP_DIRECTORY/$BACKUP + message-players "Deleted old backup" "$BACKUP" +} + # Sequential delete method delete-sequentially () { local BACKUPS=($(ls $BACKUP_DIRECTORY)) - while [[ $NUMBER_OF_BACKUPS_TO_KEEP -ge 0 && ${#BACKUPS[@]} -ge $NUMBER_OF_BACKUPS_TO_KEEP ]]; do - rm $BACKUP_DIRECTORY/${BACKUPS[0]} - message-players "Deleted old backup" "${BACKUPS[0]}" + while [[ $MAX_BACKUPS -ge 0 && ${#BACKUPS[@]} -gt $MAX_BACKUPS ]]; do + delete-backup ${BACKUPS[0]} BACKUPS=($(ls $BACKUP_DIRECTORY)) done } +# Functions to sort backups into correct categories based on timestamps +is-hourly-backup () { + local TIMESTAMP=$* + local MINUTE=$(date -d "$TIMESTAMP" +%M) + return $MINUTE +} +is-daily-backup () { + local TIMESTAMP=$* + local HOUR=$(date -d "$TIMESTAMP" +%H) + return $HOUR +} +is-weekly-backup () { + local TIMESTAMP=$* + local DAY=$(date -d "$TIMESTAMP" +%u) + return $((DAY - 1)) +} + +# Helper function to sum an array +array-sum () { + SUM=0 + for NUMBER in $*; do + (( SUM += NUMBER )) + done + echo $SUM +} + # Thinning delete method delete-thinning () { - local HOURLY_BLOCK_SIZE=96 - local DAILY_BLOCK_SIZE=30 + # sub-hourly, hourly, daily, weekly is everything else + local BLOCK_SIZES=(16 24 30) + # First block is unconditional + # The next blocks will only accept files whose names cause these functions to return true (0) + local BLOCK_FUNCTIONS=("is-hourly-backup" "is-daily-backup" "is-weekly-backup") + + # Warn if $MAX_BACKUPS does not have enough room for all the blocks + TOTAL_BLOCK_SIZE=$(array-sum ${BLOCK_SIZES[@]}) + if [[ $TOTAL_BLOCK_SIZE -gt $MAX_BACKUPS ]]; then + if ! $SUPPRESS_WARNINGS; then + log-warning "MAX_BACKUPS ($MAX_BACKUPS) is smaller than TOTAL_BLOCK_SIZE ($TOTAL_BLOCK_SIZE)" + fi + fi + + local CURRENT_INDEX=0 local BACKUPS=($(ls -r $BACKUP_DIRECTORY)) # List newest first - local OLDEST_HOURLY_BACKUP=${BACKUPS[HOURLY_BLOCK_SIZE - 1]} - local OLDEST_HOURLY_BACKUP_HOUR=${OLDEST_HOURLY_BACKUP:11:5} + for BLOCK_INDEX in ${!BLOCK_SIZES[@]}; do + local BLOCK_SIZE=${BLOCK_SIZES[BLOCK_INDEX]} + local BLOCK_FUNCTION=${BLOCK_FUNCTIONS[BLOCK_INDEX]} + local OLDEST_BACKUP_IN_BLOCK_INDEX=$((BLOCK_SIZE + CURRENT_INDEX)) # Not an off-by-one error because a new backup was already saved + local OLDEST_BACKUP_IN_BLOCK=${BACKUPS[OLDEST_BACKUP_IN_BLOCK_INDEX]} - # DEBUG: Log the oldest hourly backup to console - if $DEBUG; then - echo "Oldest hourly backup is: $OLDEST_HOURLY_BACKUP" - fi - - # If the oldest hourly backup was not at midnight, delete it - if [ "$OLDEST_HOURLY_BACKUP_HOUR" != "00-00" ] && [[ "$OLDEST_HOURLY_BACKUP" != "" ]]; then - rm $BACKUP_DIRECTORY/$OLDEST_HOURLY_BACKUP - message-players "Deleted old backup" "$OLDEST_HOURLY_BACKUP" - else - # Oldest hourly backup was at midnight, so it is now a daily backup - local OLDEST_DAILY_BACKUP=${BACKUPS[HOURLY_BLOCK_SIZE + DAILY_BLOCK_SIZE - 1]} - local OLDEST_DAILY_BACKUP_DAY=${OLDEST_DAILY_BACKUP:8:2} - - # DEBUG: Log the oldest hourly backup to console - if $DEBUG; then - echo "Oldest daily backup is: $OLDEST_DAILY_BACKUP" + if [[ $OLDEST_BACKUP_IN_BLOCK == "" ]]; then + break fi - # If the oldest daily backup was not on the 1st, delete it - if [ "$OLDEST_DAILY_BACKUP_DAY" -ne 1 ] && [[ "$OLDEST_DAILY_BACKUP" != "" ]]; then - rm $BACKUP_DIRECTORY/$OLDEST_DAILY_BACKUP - message-players "Deleted old backup" "$OLDEST_DAILY_BACKUP" + local OLDEST_BACKUP_TIMESTAMP=$(parse-file-timestamp ${OLDEST_BACKUP_IN_BLOCK:0:19}) + local BLOCK_COMMAND="$BLOCK_FUNCTION $OLDEST_BACKUP_TIMESTAMP" + + if $BLOCK_COMMAND; then + # Oldest backup in this block satisfies the condition for placement in the next block + if $DEBUG; then + echo "$OLDEST_BACKUP_IN_BLOCK promoted to next block" + fi else - # Oldest daily backup was on the 1st, so it is now a monthly backup - delete-sequentially # Delete old monthly backups + # Oldest backup in this block does not satisfy the condition for placement in next block + delete-backup $OLDEST_BACKUP_IN_BLOCK + break fi - fi + + ((CURRENT_INDEX += BLOCK_SIZE)) + done + + delete-sequentially } # Disable world autosaving @@ -115,13 +232,13 @@ execute-command "save-off" START_TIME=$(date +"%s") case $COMPRESSION_ALGORITHM in "") # No compression - tar -cf $ARCHIVE_PATH -C $SERVER_DIRECTORY world + tar -cf $ARCHIVE_PATH -C $SERVER_WORLD . ;; *) # With compression - tar -cf - -C $SERVER_DIRECTORY world | $COMPRESSION_ALGORITHM -cv -$COMPRESSION_LEVEL - > $ARCHIVE_PATH + tar -cf - -C $SERVER_WORLD . | $COMPRESSION_ALGORITHM -cv -$COMPRESSION_LEVEL - > $ARCHIVE_PATH 2>> /dev/null ;; esac -sync $ARCHIVE_PATH +sync END_TIME=$(date +"%s") # Enable world autosaving @@ -141,15 +258,15 @@ delete-old-backups () { } # Notify players of completion -WORLD_SIZE_KB=$(du --max-depth=0 $SERVER_DIRECTORY/world | awk '{print $1}') -ARCHIVE_SIZE_KB=$(du $ARCHIVE_PATH | awk '{print $1}') -COMPRESSION_PERCENT=$(($ARCHIVE_SIZE_KB * 100 / $WORLD_SIZE_KB)) +WORLD_SIZE_BYTES=$(du -b --max-depth=0 $SERVER_WORLD | awk '{print $1}') +ARCHIVE_SIZE_BYTES=$(du -b $ARCHIVE_PATH | awk '{print $1}') +COMPRESSION_PERCENT=$(($ARCHIVE_SIZE_BYTES * 100 / $WORLD_SIZE_BYTES)) ARCHIVE_SIZE=$(du -h $ARCHIVE_PATH | awk '{print $1}') BACKUP_DIRECTORY_SIZE=$(du -h --max-depth=0 $BACKUP_DIRECTORY | awk '{print $1}') TIME_DELTA=$((END_TIME - START_TIME)) # Check that archive size is not null and at least 1024 KB -if [[ "$ARCHIVE_SIZE" != "" && "$ARCHIVE_SIZE_KB" -gt 1024 ]]; then +if [[ "$ARCHIVE_SIZE" != "" && "$ARCHIVE_SIZE_BYTES" -gt 8 ]]; then message-players-success "Backup complete!" "$TIME_DELTA s, $ARCHIVE_SIZE/$BACKUP_DIRECTORY_SIZE, $COMPRESSION_PERCENT%" delete-old-backups else diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..4d54554 --- /dev/null +++ b/test.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +START_TIMESTAMP=1501484400 + +ITERATIONS=1000 +MINUTE_INTERVAL=30 +MINUTES_SINCE_START=0 + +if [[ $1 != "" ]]; then + ITERATIONS=$1 +fi + +for (( c=1; c<=$ITERATIONS; c++ )); do + TIMESTAMP=$(( START_TIMESTAMP + MINUTES_SINCE_START * 60 )) + FILE_NAME=$(date -d "@$TIMESTAMP" +%F_%H-%M-%S) + ./backup.sh -q -i /home/nicolas/privatesurvival/world -o /home/nicolas/backups -f $FILE_NAME + (( MINUTES_SINCE_START += MINUTE_INTERVAL )) +done