diff --git a/Readme.md b/Readme.md index f9066db..3b4faba 100644 --- a/Readme.md +++ b/Readme.md @@ -68,6 +68,9 @@ exit after running verify `-V` ### `-I` init repository (must be run first) +### `-Z` +run `borg compact` over given repository + ### `-C` run `borg check` over given repository @@ -119,6 +122,7 @@ All below have default values if not set in the main settings file * COMPRESSION_LEVEL: 3 * ENCRYPTION: none * FORCE_VERIFY: false + * COMPACT_INTERVAL: 1 * CHECK_INTERVAL: none * KEEP_LAST: 0 * KEEP_HOURS: 0 @@ -131,6 +135,7 @@ All module settings files can have the following prefixed with `SUB_` to overrid * SUB_BACKUP_FILE * SUB_COMPRESSION * SUB_COMPRESSION_LEVEL + * SUB_COMPACT_INTERVAL * SUB_CHECK_INTERVAL * SUB_BACKUP_SET * SUB_KEEP_LAST @@ -259,3 +264,21 @@ ZABBIX_UNKNOWN_TABLES | '' | if set, changed to -f (force) ### Control files There are no control files for zabbix settings backup + + +## File connection + +Running any of the commands below +- borg.backup.file.sh +- borg.backup.gitea.sh +- borg.backup.mysql.sh +- borg.backup.pgsql.sh +- borg.backup.zabbix.sh + +1) Run `borg.backup.functions.init.sh` (always) +2) Run `borg.backup.functions.verify.sh` (always) +3) (other code in "file" module) +4) Run `borg.backup.functions.info.sh` (always) +5) Run `borg.backup.functions.compact.sh` (not if one time tag) +6) Run `borg.backup.functions.check.sh` (not if one time tag) +7) Run `borg.backup.functions.close.sh` (always, can be called on error too) diff --git a/borg.backup.file.sh b/borg.backup.file.sh index ece9d12..38b12bd 100755 --- a/borg.backup.file.sh +++ b/borg.backup.file.sh @@ -4,7 +4,7 @@ # set last edit date + time MODULE="file"; -MODULE_VERSION="1.2.1"; +MODULE_VERSION="1.2.2"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -13,8 +13,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # include and exclude file INCLUDE_FILE="borg.backup.${MODULE}.include"; EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; -# init verify and check file +# init verify, compact and check file BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact"; BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; # lock file BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; @@ -195,7 +196,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then # for the IFS="#" to work we need to replace options spaces with exactly ONE # $(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted"; # if this is borg version >1.2 we need to run compact after prune - . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.compact.sh" "auto"; # check in auto mode . "${DIR}/borg.backup.functions.check.sh" "auto"; else diff --git a/borg.backup.functions.compact.sh b/borg.backup.functions.compact.sh index aafebfc..338c546 100644 --- a/borg.backup.functions.compact.sh +++ b/borg.backup.functions.compact.sh @@ -7,17 +7,52 @@ fi; # compact (only if BORG COMPACT is set) # only for borg 1.2 - -# reset to normal IFS, so command works here -IFS=${_IFS}; if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then - printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}"; - BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}"; - if [ ${DEBUG} -eq 1 ]; then - echo "${BORG_COMPACT}"; + RUN_COMPACT=0; + if [ $# -ge 1 ] && [ "$1" = "auto" ]; then + # strip any spaces and convert to int + COMPACT_INTERVAL=$(echo "${COMPACT_INTERVAL}" | sed -e 's/ //g'); + # not a valid compact interval, do no compact + if [ -z "${COMPACT_INTERVAL##*[!0-9]*}" ]; then + COMPACT_INTERVAL=0; + fi; + # get current date timestmap + CURRENT_DATE=$(date +%s); + if [ ${COMPACT_INTERVAL} -eq 1 ]; then + RUN_COMPACT=1; + # set new compact time here + echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}"; + elif [ ${COMPACT_INTERVAL} -gt 1 ]; then + # else load last timestamp and check if today - last time stamp > days + if [ -z "${LAST_COMPACT_DATE}" ]; then + LAST_COMPACT_DATE=$(cat "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" 2>/dev/null | sed -e 's/ //g'); + fi; + # file date is not a timestamp + if [ -z "${LAST_COMPACT_DATE##*[!0-9]*}" ]; then + LAST_COMPACT_DATE=0; + fi; + # if the difference greate than compact date, run. COMPACT INTERVAL is in days + if [ $(($CURRENT_DATE-$LAST_COMPACT_DATE)) -ge $((${COMPACT_INTERVAL}*86400)) ]; then + RUN_COMPACT=1; + # set new compact time here + echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}"; + fi; + fi; + elif [ ${COMPACT} -eq 1 ]; then + RUN_COMPACT=1; fi; - if [ ${DRYRUN} -eq 0 ]; then - ${BORG_COMPACT}; + + if [ ${RUN_COMPACT} -eq 1 ]; then + # reset to normal IFS, so command works here + IFS=${_IFS}; + printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}"; + BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}"; + if [ ${DEBUG} -eq 1 ]; then + echo "${BORG_COMPACT}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + ${BORG_COMPACT}; + fi; fi; fi; diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index 2d4e9c0..c19092a 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -24,7 +24,7 @@ function version { } # version for all general files -VERSION="4.3.0"; +VERSION="4.4.0"; # borg version and borg comamnd BORG_VERSION=""; @@ -49,7 +49,9 @@ EXCLUDE_FILE=""; # backup folder initialzed verify BACKUP_INIT_FILE=""; BACKUP_INIT_DATE=""; -# fiel with last check timestamp +# file with last compact date +BACKUP_COMPACT_FILE=""; +# file with last check timestamp BACKUP_CHECK_FILE=""; # one time backup prefix tag, if set will use .-Y-M-DTh:m:s type backup prefix ONE_TIME_TAG=""; @@ -65,6 +67,7 @@ INFO=0; VERIFY=0; CHECK=0; CHECK_VERIFY_DATA=0; +COMPACT=0; INIT=0; EXIT=0; PRINT=0; @@ -88,7 +91,7 @@ INIT_REPOSITORY=0; FOLDER_OK=0; TMP_EXCLUDE_FILE=""; # printf strings -PRINTF_INFO_STRING="%-21s: %s\n"; +PRINTF_INFO_STRING="%-23s: %s\n"; PRINTF_MASTER_BLOCK="=== [%-8s: %19s] ==[%s]====================================>\n"; PRINTF_SUB_BLOCK="|-- [%-8s: %19s] --[%s]------------------------------------>\n"; PRINTF_SUBEXT_BLOCK="|-- [%-8s: %s: %19s] --[%s]------------------------------------>\n"; @@ -127,6 +130,11 @@ ENCRYPTION=""; DEFAULT_FORCE_VERIFY="false"; FORCE_VERIFY=""; FORCE_CHECK=""; # Deprecated name, use FORCE_VERIFY +# compact +DEFAULT_COMPACT_INTERVAL="1"; +COMPACT_INTERVAL=""; +SUB_COMPACT_INTERVAL=""; +# check # default interval is none DEFAULT_CHECK_INTERVAL=""; CHECK_INTERVAL=""; @@ -192,6 +200,7 @@ function usage() -T : create one time stand alone backup prefixed with tag name -D : remove a tagged backup set, full name must be given -b : override default path + -Z: run compress after prune/backup. Only for borg 1.2 or newer -C: run borg check if repository is ok -y: in combination with -C: add --verify-data -p : in combinatio with -C: only check archives with prefix or glob @@ -213,7 +222,7 @@ function usage() } # set options -while getopts ":c:L:T:D:b:p:vldniCVeIPyh" opt; do +while getopts ":c:L:T:D:b:p:vldniCVeIPyZh" opt; do case "${opt}" in c|config) BASE_FOLDER=${OPTARG}; @@ -230,6 +239,10 @@ while getopts ":c:L:T:D:b:p:vldniCVeIPyh" opt; do b|borg) OPT_BORG_EXECUTEABLE=${OPTARG}; ;; + Z|Compact) + # will run compact alone + COMPACT=1; + ;; C|Check) # will run borg check # alt modes --repository-only, --archives-only, @@ -422,6 +435,10 @@ fi; if [ -z "${ENCRYPTION}" ]; then ENCRYPTION="${DEFAULT_ENCRYPTION}"; fi; +# check interval override +if [ -z "${COMPACT_INTERVAL}" ]; then + COMPACT_INTERVAL="${DEFAULT_COMPACT_INTERVAL}"; +fi; if [ -z "${CHECK_INTERVAL}" ]; then CHECK_INTERVAL="${DEFAULT_CHECK_INTERVAL}"; fi; @@ -471,6 +488,10 @@ if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then if [ ! -z "${SUB_COMPRESSION_LEVEL}" ]; then COMPRESSION_LEVEL=${SUB_COMPRESSION_LEVEL}; fi; + # compact interval override + if [ ! -z "${SUB_COMPACT_INTERVAL}" ]; then + COMPACT_INTERVAL="${SUB_COMPACT_INTERVAL}"; + fi; # override check interval if [ ! -z "${SUB_CHECK_INTERVAL}" ]; then CHECK_INTERVAL="${SUB_CHECK_INTERVAL}"; @@ -537,6 +558,7 @@ if [[ ${TARGET_FOLDER} =~ ^~\/ ]]; then exit 1; fi +# COMPACT_INTERVAL must be a number from -1 to 365 # CHECK_INTERVAL must be a number from -1 to 365 # log file set and check diff --git a/borg.backup.functions.verify.sh b/borg.backup.functions.verify.sh index 67c4313..dfb8e7b 100644 --- a/borg.backup.functions.verify.sh +++ b/borg.backup.functions.verify.sh @@ -12,15 +12,6 @@ BACKUP_INIT_DATE=''; if [ -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then BACKUP_INIT_DATE=$(printf '%(%c)T' $(cat "${BASE_FOLDER}${BACKUP_INIT_FILE}" 2>/dev/null)); fi; -# last check date if set -BACKUP_LAST_CHECK_DATE=''; -LAST_CHECK_DATE=''; -CONVERT_TIME=''; -if [ -f "${BASE_FOLDER}${BACKUP_CHECK_FILE}" ]; then - LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null); - BACKUP_LAST_CHECK_DATE=$(printf '%(%c)T' ${LAST_CHECK_DATE}); - CONVERT_TIME=$(convert_time $(($(date +%s)-${LAST_CHECK_DATE}))); -fi; # start logging from here exec &> >(tee -a "${LOG}"); printf "${PRINTF_MASTER_BLOCK}" "START" "$(date +'%F %T')" "${MODULE}"; @@ -37,10 +28,31 @@ printf "${PRINTF_INFO_STRING}" "Hostname" "${HOSTNAME}"; printf "${PRINTF_INFO_STRING}" "Base folder" "${BASE_FOLDER}"; # Module init date (when init file was writen) printf "${PRINTF_INFO_STRING}" "Module init date" "${BACKUP_INIT_DATE}"; +# print last compact date if positive integer +# only if borg > 1.2 +if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then + if [ "${COMPACT_INTERVAL##*[!0-9]*}" ]; then + printf "${PRINTF_INFO_STRING}" "Module compact interval" "${COMPACT_INTERVAL}"; + if [ -f "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" ]; then + LAST_COMPACT_DATE=$(cat "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" 2>/dev/null); + printf "${PRINTF_INFO_STRING}" "Module last compact" \ + "$(printf '%(%c)T' ${LAST_COMPACT_DATE}) ($(convert_time $(($(date +%s)-${LAST_COMPACT_DATE}))) ago)"; + else + printf "${PRINTF_INFO_STRING}" "Module last compact" "No compact run yet" + fi; + fi; +fi; # print last check date if positive integer if [ "${CHECK_INTERVAL##*[!0-9]*}" ]; then printf "${PRINTF_INFO_STRING}" "Module check interval" "${CHECK_INTERVAL}"; - printf "${PRINTF_INFO_STRING}" "Module last check" "${BACKUP_LAST_CHECK_DATE} (${CONVERT_TIME} ago)"; + # get last check date + if [ -f "${BASE_FOLDER}${BACKUP_CHECK_FILE}" ]; then + LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null); + printf "${PRINTF_INFO_STRING}" "Module last check" \ + "$(printf '%(%c)T' ${LAST_CHECK_DATE}) ($(convert_time $(($(date +%s)-${LAST_CHECK_DATE}))) ago)"; + else + printf "${PRINTF_INFO_STRING}" "Module last check" "No check run yet"; + fi; fi; # if force verify is true set VERIFY to 1 unless INFO is 1 @@ -376,6 +388,13 @@ if [ ${PRINT} -eq 1 ]; then exit; fi; +# run borg compact command and exit +if [ ${COMPACT} -eq 1 ]; then + . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.close.sh"; + exit; +fi; + # run borg check command and exit if [ ${CHECK} -eq 1 ]; then . "${DIR}/borg.backup.functions.check.sh"; diff --git a/borg.backup.gitea.sh b/borg.backup.gitea.sh index 076e158..bca7bbe 100755 --- a/borg.backup.gitea.sh +++ b/borg.backup.gitea.sh @@ -3,15 +3,16 @@ # Backup gitea database, all git folders and gitea settings MODULE="gitea" -MODULE_VERSION="1.1.3"; +MODULE_VERSION="1.1.4"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # init system . "${DIR}/borg.backup.functions.init.sh"; -# init verify and check file +# init verify, compact and check file BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact"; BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; # lock file BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; @@ -83,7 +84,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; # if this is borg version >1.2 we need to run compact after prune - . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.compact.sh" "auto"; # check in auto mode . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh index 4ba4dbc..7ab272b 100755 --- a/borg.backup.mysql.sh +++ b/borg.backup.mysql.sh @@ -7,7 +7,7 @@ # set last edit date + time MODULE="mysql" -MODULE_VERSION="1.1.1"; +MODULE_VERSION="1.1.2"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -18,8 +18,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi INCLUDE_FILE="borg.backup.${MODULE}.include"; EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only"; -# init verify and check file +# init verify, compact and check file BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact"; BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; # lock file BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; @@ -217,7 +218,7 @@ fi; # run compact at the end if not a dry run if [ -z "${ONE_TIME_TAG}" ]; then # if this is borg version >1.2 we need to run compact after prune - . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.compact.sh" "auto"; # check in auto mode . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index 89a3b84..deb5480 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -7,7 +7,7 @@ # set last edit date + time MODULE="pgsql" -MODULE_VERSION="1.2.1"; +MODULE_VERSION="1.2.2"; DIR="${BASH_SOURCE%/*}" @@ -20,8 +20,9 @@ INCLUDE_FILE="borg.backup.${MODULE}.include"; EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only"; DATA_ONLY_FILE="borg.backup.${MODULE}.data-only"; -# init verify and check file +# init verify, compact and check file BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact"; BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; # lock file BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; @@ -269,7 +270,7 @@ fi; # run compact at the end if not a dry run if [ -z "${ONE_TIME_TAG}" ]; then # if this is borg version >1.2 we need to run compact after prune - . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.compact.sh" "auto"; # check in auto mode . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; diff --git a/borg.backup.settings-default b/borg.backup.settings-default index 9950f49..ced6f92 100644 --- a/borg.backup.settings-default +++ b/borg.backup.settings-default @@ -27,8 +27,14 @@ COMPRESSION_LEVEL=""; # Blank passwords allowed for only key (if used, use keyfile) # See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically ENCRYPTION=""; -# force repository verirfy, default is off, set to true for verify on every run +# force repository verify, default is off, set to true for verify on every run FORCE_VERIFY=""; +# compact interval, only if using borg 1.2 or higher +# after how many days to run compress on data +# default is 1 for run after each prune +# set to 0 or negative to turn off +# or any other value for every n days +COMPACT_INTERVAL=""; # check interval, if 0 or negative number, no check will ever run # if empty fall back to default set # if set to 1 then every time the script runs diff --git a/borg.backup.zabbix.sh b/borg.backup.zabbix.sh index 2cb6baf..ff6cc99 100755 --- a/borg.backup.zabbix.sh +++ b/borg.backup.zabbix.sh @@ -3,15 +3,16 @@ # Backup zabbix config and settings only MODULE="zabbix" -MODULE_VERSION="1.1.2"; +MODULE_VERSION="1.1.3"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # init system . "${DIR}/borg.backup.functions.init.sh"; -# init verify and check file +# init verify, compact and check file BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact"; BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; # lock file BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; @@ -81,7 +82,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then printf "${PRINTF_SUB_BLOCK}" "PRUNE" "$(date +'%F %T')" "${MODULE}"; ${BORG_PRUNE}; # if this is borg version >1.2 we need to run compact after prune - . "${DIR}/borg.backup.functions.compact.sh"; + . "${DIR}/borg.backup.functions.compact.sh" "auto"; # check in auto mode . "${DIR}/borg.backup.functions.check.sh" "auto"; fi;