diff --git a/Readme.md b/Readme.md index cc8d753..f9066db 100644 --- a/Readme.md +++ b/Readme.md @@ -63,7 +63,7 @@ print list of archives created verify if repository exists, if not abort ### `-e` -exit after check +exit after running verify `-V` ### `-I` init repository (must be run first) @@ -118,7 +118,8 @@ All below have default values if not set in the main settings file * COMPRESSION: zstd * COMPRESSION_LEVEL: 3 * ENCRYPTION: none - * FORCE_CHECK: false + * FORCE_VERIFY: false + * CHECK_INTERVAL: none * KEEP_LAST: 0 * KEEP_HOURS: 0 * KEEP_DAYS: 7 @@ -130,6 +131,7 @@ All module settings files can have the following prefixed with `SUB_` to overrid * SUB_BACKUP_FILE * SUB_COMPRESSION * SUB_COMPRESSION_LEVEL + * SUB_CHECK_INTERVAL * SUB_BACKUP_SET * SUB_KEEP_LAST * SUB_KEEP_HOURS @@ -157,6 +159,14 @@ and `TARGET_BORG_PATH="";` if the target borg is in a non default path `BORG_EXECUTABLE=""` +## Note on CHECK_INTERVAL and SUB_CHECK_INTERVAL + +If set to empty or 0 it will not run an automatic check. If set to 1 it will run a check after each backup. Any other value means days differente to the last check. + +Running check manually (`-C`) will not reset the last check timestamp. + +Automatic checks always add `--verify-data`, with manual `-C` the option `-y` has to be set. + ## File backup settings On new setups it is recommended to use the `borg.backup.file.setings` and set diff --git a/borg.backup.file.sh b/borg.backup.file.sh index bf5f7b8..ece9d12 100755 --- a/borg.backup.file.sh +++ b/borg.backup.file.sh @@ -11,10 +11,13 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi . "${DIR}/borg.backup.functions.init.sh"; # include and exclude file -INCLUDE_FILE="borg.backup.file.include"; -EXCLUDE_FILE="borg.backup.file.exclude"; -# init verify file -BACKUP_INIT_VERIFY="borg.backup.file.init"; +INCLUDE_FILE="borg.backup.${MODULE}.include"; +EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; +# init verify and check file +BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; +# lock file +BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # verify valid data . "${DIR}/borg.backup.functions.verify.sh"; @@ -25,7 +28,7 @@ if [ ! -f "${BASE_FOLDER}${INCLUDE_FILE}" ]; then . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; -echo "--- [INCLUDE: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; +printf "${PRINTF_SUB_BLOCK}" "INCLUDE" "$(date +'%F %T')" "${MODULE}"; # folders to backup FOLDERS=(); # this if for debug output with quoted folders @@ -90,7 +93,7 @@ done<"${BASE_FOLDER}${INCLUDE_FILE}"; # exclude list if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then - echo "--- [EXCLUDE: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "EXCLUDE" "$(date +'%F %T')" "${MODULE}"; # check that the folders in that exclude file are actually valid, # remove non valid ones and warn #TMP_EXCLUDE_FILE=$(mktemp --tmpdir ${EXCLUDE_FILE}.XXXXXXXX); # non mac @@ -159,7 +162,7 @@ COMMAND=${COMMAND}${REPOSITORY}::${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET . "${DIR}/borg.backup.functions.info.sh"; if [ $FOLDER_OK -eq 1 ]; then - echo "--- [BACKUP: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "BACKUP" "$(date +'%F %T')" "${MODULE}"; # show command if [ ${DEBUG} -eq 1 ]; then echo $(echo ${COMMAND} | sed -e 's/[ ][ ]*/ /g') ${FOLDERS_Q[*]}; @@ -182,7 +185,7 @@ fi; # clean up, always verbose, but only if we do not run one time tag if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "PRUNE" "$(date +'%F %T')" "${MODULE}"; # build command COMMAND="${BORG_COMMAND} prune ${OPT_REMOTE} -v --list ${OPT_PROGRESS} ${DRY_RUN_STATS} -P ${BACKUP_SET_PREFIX} ${KEEP_OPTIONS[*]} ${REPOSITORY}"; echo "Prune repository with keep${KEEP_INFO:1}"; @@ -193,6 +196,8 @@ if [ -z "${ONE_TIME_TAG}" ]; then $(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted"; # if this is borg version >1.2 we need to run compact after prune . "${DIR}/borg.backup.functions.compact.sh"; + # check in auto mode + . "${DIR}/borg.backup.functions.check.sh" "auto"; else echo "[#] No prune with tagged backup"; fi; diff --git a/borg.backup.functions.check.sh b/borg.backup.functions.check.sh new file mode 100644 index 0000000..b419d30 --- /dev/null +++ b/borg.backup.functions.check.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + +# run borg check (NOT REPAIR) + +RUN_CHECK=0; +if [ $# -ge 1 ] && [ "$1" = "auto" ]; then + # strip any spaces and convert to int + CHECK_INTERVAL=$(echo "${CHECK_INTERVAL}" | sed -e 's/ //g'); + # not a valid check interval, do no check + if [ -z "${CHECK_INTERVAL##*[!0-9]*}" ]; then + CHECK_INTERVAL=0; + fi; + # get current date timestmap + CURRENT_DATE=$(date +%s); + # if =1 always ok + if [ ${CHECK_INTERVAL} -eq 1 ]; then + RUN_CHECK=1; + # always add verify data for automatic check + OPT_CHECK_VERIFY_DATA="--verify-data"; + # set new check time here + echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_CHECK_FILE}"; + elif [ ${CHECK_INTERVAL} -gt 1 ]; then + # else load last timestamp and check if today - last time stamp > days + if [ -z "${LAST_CHECK_DATE}" ]; then + LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null | sed -e 's/ //g'); + fi; + # file date is not a timestamp + if [ -z "${LAST_CHECK_DATE##*[!0-9]*}" ]; then + LAST_CHECK_DATE=0; + fi; + # if the difference greate than check date, run. CHECK INTERVAL is in days + if [ $(($CURRENT_DATE-$LAST_CHECK_DATE)) -ge $((${CHECK_INTERVAL}*86400)) ]; then + RUN_CHECK=1; + # always add verify data for automatic check + OPT_CHECK_VERIFY_DATA="--verify-data"; + # set new check time here + echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_CHECK_FILE}"; + fi; + fi; +elif [ ${CHECK} -eq 1 ]; then + RUN_CHECK=1; +fi; + +if [ ${RUN_CHECK} -eq 1 ]; then + # run borg check command + IFS=${_IFS}; + printf "${PRINTF_SUB_BLOCK}" "CHECK" "$(date +'%F %T')" "${MODULE}"; + # repare command + OPT_GLOB=""; + if [[ "${CHECK_PREFIX}" =~ $REGEX_GLOB ]]; then + OPT_GLOB="-a '${CHECK_PREFIX}'" + elif [ ! -z "${CHECK_PREFIX}" ]; then + OPT_GLOB="-P ${CHECK_PREFIX}"; + fi; + # debug/dryrun + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}"; + fi; + # run info command if not a dry drun + if [ ${DRYRUN} -eq 0 ]; then + # if glob add glob command directly + if [[ "${CHECK_PREFIX}" =~ $REGEX_GLOB ]]; then + ${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} -a "${CHECK_PREFIX}" ${REPOSITORY}; + else + ${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}; + fi; + fi; + # print additional info for use --repair command + # but only for manual checks + if [ ${VERBOSE} -eq 1 ] && [ ${CHECK} -eq 1 ]; then + echo ""; + echo "In case of needed repair: " + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} --repair ${OPT_GLOB} ${REPOSITORY}"; + echo "Before running repair, a copy from the backup should be made because repair might damage a backup" + fi; +fi; + +# __END__ diff --git a/borg.backup.functions.close.sh b/borg.backup.functions.close.sh index da51b50..6cb2e77 100644 --- a/borg.backup.functions.close.sh +++ b/borg.backup.functions.close.sh @@ -1,14 +1,24 @@ #!/usr/bin/env bash +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + # unset borg settings unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK # error abort without duration and error notice if [ $# -ge 1 ] && [ "$1" = "1" ]; then - echo "=== [ERROR: $(date +'%F %T')] ==[${MODULE}]====================================>"; + printf "${PRINTF_MASTER_BLOCK}" "ERROR" "$(date +'%F %T')" "${MODULE}"; else + # delete lock file + if [ -f "${BASE_FOLDER}${BACKUP_LOCK_FILE}" ]; then + rm "${BASE_FOLDER}${BACKUP_LOCK_FILE}"; + fi; + # running time calculation DURATION=$[ $(date +'%s')-$START ]; echo "=== [Run time: $(convert_time ${DURATION})]"; - echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + printf "${PRINTF_MASTER_BLOCK}" "END" "$(date +'%F %T')" "${MODULE}"; fi; # __END__ diff --git a/borg.backup.functions.compact.sh b/borg.backup.functions.compact.sh index ca2ff3c..aafebfc 100644 --- a/borg.backup.functions.compact.sh +++ b/borg.backup.functions.compact.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + # compact (only if BORG COMPACT is set) # only for borg 1.2 # reset to normal IFS, so command works here IFS=${_IFS}; if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then - echo "--- [COMPACT:$(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}"; BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}"; if [ ${DEBUG} -eq 1 ]; then echo "${BORG_COMPACT}"; diff --git a/borg.backup.functions.info.sh b/borg.backup.functions.info.sh index 2b0ebdc..e18eed2 100644 --- a/borg.backup.functions.info.sh +++ b/borg.backup.functions.info.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + if [ ${INFO} -eq 1 ]; then - echo "--- [INFO : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "INFO" "$(date +'%F %T')" "${MODULE}"; # show command on debug or dry run if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}"; diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index 6b3bb96..2d4e9c0 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + set -ETu #-e -o pipefail trap cleanup SIGINT SIGTERM ERR @@ -42,8 +47,10 @@ SETTINGS_FILE="borg.backup.settings"; INCLUDE_FILE=""; EXCLUDE_FILE=""; # backup folder initialzed verify -BACKUP_INIT_VERIFY=""; +BACKUP_INIT_FILE=""; BACKUP_INIT_DATE=""; +# fiel with last check timestamp +BACKUP_CHECK_FILE=""; # one time backup prefix tag, if set will use .-Y-M-DTh:m:s type backup prefix ONE_TIME_TAG=""; DELETE_ONE_TIME_TAG=""; @@ -80,6 +87,12 @@ PRUNE_DEBUG=""; INIT_REPOSITORY=0; FOLDER_OK=0; TMP_EXCLUDE_FILE=""; +# printf strings +PRINTF_INFO_STRING="%-21s: %s\n"; +PRINTF_MASTER_BLOCK="=== [%-8s: %19s] ==[%s]====================================>\n"; +PRINTF_SUB_BLOCK="|-- [%-8s: %19s] --[%s]------------------------------------>\n"; +PRINTF_SUBEXT_BLOCK="|-- [%-8s: %s: %19s] --[%s]------------------------------------>\n"; +PRINTF_DB_SUB_BLOCK=">>- [%-8s: %s] =======================[%s]====================================>\n"; # opt flags OPT_VERBOSE=""; OPT_PROGRESS=""; @@ -114,6 +127,11 @@ ENCRYPTION=""; DEFAULT_FORCE_VERIFY="false"; FORCE_VERIFY=""; FORCE_CHECK=""; # Deprecated name, use FORCE_VERIFY +# default interval is none +DEFAULT_CHECK_INTERVAL=""; +CHECK_INTERVAL=""; +SUB_CHECK_INTERVAL=""; +# backup set names BACKUP_SET=""; SUB_BACKUP_SET=""; # for database backup only @@ -404,6 +422,9 @@ fi; if [ -z "${ENCRYPTION}" ]; then ENCRYPTION="${DEFAULT_ENCRYPTION}"; fi; +if [ -z "${CHECK_INTERVAL}" ]; then + CHECK_INTERVAL="${DEFAULT_CHECK_INTERVAL}"; +fi; # deprecated name FORCE_CHECK, use FORCE_VERIFY instead if [ ! -z "${FORCE_CHECK}" ]; then FORCE_VERIFY="${FORCE_CHECK}"; @@ -450,6 +471,10 @@ if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then if [ ! -z "${SUB_COMPRESSION_LEVEL}" ]; then COMPRESSION_LEVEL=${SUB_COMPRESSION_LEVEL}; fi; + # override check interval + if [ ! -z "${SUB_CHECK_INTERVAL}" ]; then + CHECK_INTERVAL="${SUB_CHECK_INTERVAL}"; + fi; # check override for keep time if [ ! -z "${SUB_KEEP_LAST}" ]; then KEEP_LAST=${SUB_KEEP_LAST}; @@ -512,6 +537,8 @@ if [[ ${TARGET_FOLDER} =~ ^~\/ ]]; then exit 1; fi +# CHECK_INTERVAL must be a number from -1 to 365 + # log file set and check # option folder overrides all other folders if [ ! -z "${OPT_LOG_FOLDER}" ]; then diff --git a/borg.backup.functions.verify.sh b/borg.backup.functions.verify.sh index 3101f9a..67c4313 100644 --- a/borg.backup.functions.verify.sh +++ b/borg.backup.functions.verify.sh @@ -1,25 +1,47 @@ #!/usr/bin/env bash +if [ -z "${MODULE}" ]; then + echo "Script cannot be run on its own"; + exit 1; +fi; + # start time in seconds START=$(date +'%s'); # set init date, or today if not file is set -BACKUP_INIT_DATE=$(printf '%(%c)T' $(cat "${BASE_FOLDER}${BACKUP_INIT_VERIFY}" 2>/dev/null)); +BACKUP_INIT_DATE=''; +if [ -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then + BACKUP_INIT_DATE=$(printf '%(%c)T' $(cat "${BASE_FOLDER}${BACKUP_INIT_FILE}" 2>/dev/null)); +fi; +# last check date if set +BACKUP_LAST_CHECK_DATE=''; +LAST_CHECK_DATE=''; +CONVERT_TIME=''; +if [ -f "${BASE_FOLDER}${BACKUP_CHECK_FILE}" ]; then + LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null); + BACKUP_LAST_CHECK_DATE=$(printf '%(%c)T' ${LAST_CHECK_DATE}); + CONVERT_TIME=$(convert_time $(($(date +%s)-${LAST_CHECK_DATE}))); +fi; # start logging from here exec &> >(tee -a "${LOG}"); -echo "=== [START : $(date +'%F %T')] ==[${MODULE}]====================================>"; +printf "${PRINTF_MASTER_BLOCK}" "START" "$(date +'%F %T')" "${MODULE}"; # show info for version always -echo "Script version : ${VERSION}"; +printf "${PRINTF_INFO_STRING}" "Script version" "${VERSION}"; # show type -echo "Backup module : ${MODULE}"; -echo "Module version : ${MODULE_VERSION}"; +printf "${PRINTF_INFO_STRING}" "Backup module" "${MODULE}"; +printf "${PRINTF_INFO_STRING}" "Module version" "${MODULE_VERSION}"; # borg version -echo "Borg version : ${BORG_VERSION}"; +printf "${PRINTF_INFO_STRING}" "Borg version" "${BORG_VERSION}"; # host name -echo "Hostname : ${HOSTNAME}"; +printf "${PRINTF_INFO_STRING}" "Hostname" "${HOSTNAME}"; # show base folder always -echo "Base folder : ${BASE_FOLDER}"; +printf "${PRINTF_INFO_STRING}" "Base folder" "${BASE_FOLDER}"; # Module init date (when init file was writen) -echo "Module init date: ${BACKUP_INIT_DATE}"; +printf "${PRINTF_INFO_STRING}" "Module init date" "${BACKUP_INIT_DATE}"; +# print last check date if positive integer +if [ "${CHECK_INTERVAL##*[!0-9]*}" ]; then + printf "${PRINTF_INFO_STRING}" "Module check interval" "${CHECK_INTERVAL}"; + printf "${PRINTF_INFO_STRING}" "Module last check" "${BACKUP_LAST_CHECK_DATE} (${CONVERT_TIME} ago)"; +fi; # if force verify is true set VERIFY to 1 unless INFO is 1 # Needs bash 4.0 at lesat for this @@ -82,7 +104,7 @@ elif [ ! -z "${TARGET_HOST}" ]; then fi; # we dont allow special characters, so we don't need to special escape it REPOSITORY="${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}"; -echo "Repository : ${REPOSITORY}"; +printf "${PRINTF_INFO_STRING}" "Repository" "${REPOSITORY}"; # check if given compression name and level are valid OPT_COMPRESSION=''; @@ -205,6 +227,22 @@ else fi; fi; +# check if we have lock file, check pid in lock file, if no matching pid found +# running remove lock file +if [ -f "${BASE_FOLDER}${BACKUP_LOCK_FILE}" ]; then + LOCK_PID=$(cat "${BASE_FOLDER}${BACKUP_LOCK_FILE}" 2>/dev/null); + # check if lock file pid has an active program attached to it + if [ -f /proc/${LOCK_PID}/cmdline ]; then + echo "Script is already running on PID: ${$}"; + . "${DIR}/borg.backup.functions.close.sh" 1; + exit 1; + else + echo "[#] Clean up stale lock file for PID: ${LOCK_PID}"; + rm "${BASE_FOLDER}${BACKUP_LOCK_FILE}"; + fi; +fi; +echo "${$}" > "${BASE_FOLDER}${BACKUP_LOCK_FILE}"; + # for folders list split set to "#" and keep the old setting as is _IFS=${IFS}; IFS="#"; @@ -239,7 +277,7 @@ COMMAND_INFO="${COMMAND_EXPORT}${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}" # else a normal verify is ok # unless explicit given, verify is skipped if [ ${VERIFY} -eq 1 ] || [ ${INIT} -eq 1 ]; then - echo "--- [VERIFY: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "VERIFY" "$(date +'%F %T')" "${MODULE}"; if [ ! -z "${TARGET_SERVER}" ]; then if [ ${DEBUG} -eq 1 ]; then echo "${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\""; @@ -256,10 +294,10 @@ if [ ${VERIFY} -eq 1 ] || [ ${INIT} -eq 1 ]; then fi; # if verrify but no init and repo is there but init file is missing set it if [ ${VERIFY} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 0 ] && - [ ! -f "${BASE_FOLDER}${BACKUP_INIT_VERIFY}" ]; then + [ ! -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then # write init file echo "[!] Add missing init verify file"; - echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_VERIFY}"; + echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_FILE}"; fi; # end if verified but repository is not here if [ ${VERIFY} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then @@ -276,7 +314,7 @@ if [ ${VERIFY} -eq 1 ] || [ ${INIT} -eq 1 ]; then fi; fi; if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then - echo "--- [INIT : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "INIT" "$(date +'%F %T')" "${MODULE}"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}"; fi @@ -284,7 +322,7 @@ if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then # should trap and exit properly here ${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}; # write init file - echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_VERIFY}"; + echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_FILE}"; echo "Repository initialized"; echo "For more information run:" echo "${COMMAND_INFO}"; @@ -301,7 +339,7 @@ elif [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 0 ]; then fi; # verify for init file -if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_VERIFY}" ]; then +if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then echo "[! $(date +'%F %T')] It seems the repository has never been initialized." echo "Please run -I to initialize or if already initialzed run with -C for init update." . "${DIR}/borg.backup.functions.close.sh" 1; @@ -310,7 +348,7 @@ fi; # PRINT OUT current data, only do this if REPO exists if [ ${PRINT} -eq 1 ]; then - echo "--- [PRINT : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "PRINT" "$(date +'%F %T')" "${MODULE}"; FORMAT="{archive:<45} {comment:6} {start} - {end} [{id}] ({username}@{hostname}){NL}" # show command on debug or dry run if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then @@ -338,43 +376,16 @@ if [ ${PRINT} -eq 1 ]; then exit; fi; -# run borg check command +# run borg check command and exit if [ ${CHECK} -eq 1 ]; then - echo "--- [CHECK : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; - # repare command - OPT_GLOB=""; - if [[ "${CHECK_PREFIX}" =~ $REGEX_GLOB ]]; then - OPT_GLOB="-a '${CHECK_PREFIX}'" - elif [ ! -z "${CHECK_PREFIX}" ]; then - OPT_GLOB="-P ${CHECK_PREFIX}"; - fi; - # debug/dryrun - if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}"; - fi; - # run info command if not a dry drun - if [ ${DRYRUN} -eq 0 ]; then - # if glob add glob command directly - if [[ "${CHECK_PREFIX}" =~ $REGEX_GLOB ]]; then - ${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} -a "${CHECK_PREFIX}" ${REPOSITORY}; - else - ${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}; - fi; - fi; - # print additional info for use --repair command - if [ ${VERBOSE} -eq 1 ]; then - echo ""; - echo "In case of needed repair: " - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} --repair ${OPT_GLOB} ${REPOSITORY}"; - echo "Before running repair, a copy from the backup should be made because repair might damage a backup" - fi; + . "${DIR}/borg.backup.functions.check.sh"; . "${DIR}/borg.backup.functions.close.sh"; exit; fi; # DELETE ONE TIME TAG if [ ! -z "${DELETE_ONE_TIME_TAG}" ]; then - echo "--- [DELETE: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "DELETE" "$(date +'%F %T')" "${MODULE}"; # if a "*" is inside we don't do ONE archive, but globbing via -a option DELETE_ARCHIVE="" OPT_GLOB=""; diff --git a/borg.backup.gitea.sh b/borg.backup.gitea.sh index 4e3891a..96d90ec 100755 --- a/borg.backup.gitea.sh +++ b/borg.backup.gitea.sh @@ -10,8 +10,11 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # init system . "${DIR}/borg.backup.functions.init.sh"; -# init verify file -BACKUP_INIT_VERIFY="borg.backup.gitea.init"; +# init verify and check file +BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; +# lock file +BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # verify valid data . "${DIR}/borg.backup.functions.verify.sh"; @@ -54,7 +57,7 @@ BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); -echo "--- [git data and database: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; +printf "${PRINTF_SUB_BLOCK}" "BACKUP: git data and database" "$(date +'%F %T')" "${MODULE}"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}"; if [ -z "${ONE_TIME_TAG}" ]; then @@ -76,11 +79,13 @@ if [ ${DRYRUN} -eq 0 ]; then ) | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' # remove all ESC strings fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "PRUNE" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; # if this is borg version >1.2 we need to run compact after prune . "${DIR}/borg.backup.functions.compact.sh"; + # check in auto mode + . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh index 7af620b..4ba4dbc 100755 --- a/borg.backup.mysql.sh +++ b/borg.backup.mysql.sh @@ -15,11 +15,14 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi . "${DIR}/borg.backup.functions.init.sh"; # include and exclude file -INCLUDE_FILE="borg.backup.mysql.include"; -EXCLUDE_FILE="borg.backup.mysql.exclude"; -SCHEMA_ONLY_FILE="borg.backup.mysql.schema-only"; -# init verify file -BACKUP_INIT_VERIFY="borg.backup.mysql.init"; +INCLUDE_FILE="borg.backup.${MODULE}.include"; +EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; +SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only"; +# init verify and check file +BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; +# lock file +BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # verify valid data . "${DIR}/borg.backup.functions.verify.sh"; @@ -89,7 +92,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='--no-data'; schema_flag='schema'; fi; - echo "---[BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}"; # We only do a full backup and not per table backup here # Filename FILENAME="all-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql" @@ -116,15 +119,15 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then fi; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "PRUNE" "all databases" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; else ${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" | while read db; do - echo "========[DB: ${db}]========================[${MODULE}]====================================>"; - echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_DB_SUB_BLOCK}" "DB" "${db}" "${MODULE}"; + printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}"; # exclude checks include=0; if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then @@ -202,7 +205,7 @@ else fi; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "PRUNE" "${db}" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; @@ -215,6 +218,8 @@ fi; if [ -z "${ONE_TIME_TAG}" ]; then # if this is borg version >1.2 we need to run compact after prune . "${DIR}/borg.backup.functions.compact.sh"; + # check in auto mode + . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index ae6361b..89a3b84 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -16,12 +16,15 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi . "${DIR}/borg.backup.functions.init.sh"; # include and exclude file -INCLUDE_FILE="borg.backup.pgsql.include"; -EXCLUDE_FILE="borg.backup.pgsql.exclude"; -SCHEMA_ONLY_FILE="borg.backup.pgsql.schema-only"; -DATA_ONLY_FILE="borg.backup.pgsql.data-only"; -# init verify file -BACKUP_INIT_VERIFY="borg.backup.pgsql.init"; +INCLUDE_FILE="borg.backup.${MODULE}.include"; +EXCLUDE_FILE="borg.backup.${MODULE}.exclude"; +SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only"; +DATA_ONLY_FILE="borg.backup.${MODULE}.data-only"; +# init verify and check file +BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; +# lock file +BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # verify valid data . "${DIR}/borg.backup.functions.verify.sh"; @@ -98,7 +101,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='-s'; schema_flag='schema'; fi; - echo "--- [BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}"; # Filename FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" # backup set: @@ -124,7 +127,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then fi; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "PRUNE" "all databases" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; @@ -132,7 +135,7 @@ else # dump globals first db="pg_globals"; schema_flag="data"; - echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}"; # Filename FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" # backup set: @@ -158,7 +161,7 @@ else fi; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "PRUNE" "${db}" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; @@ -169,8 +172,8 @@ else owner=$(echo ${owner_db} | cut -d "," -f 1); db=$(echo ${owner_db} | cut -d "," -f 2); encoding=$(echo ${owner_db} | cut -d "," -f 3); - echo "========[DB: ${db}]========================[${MODULE}]====================================>"; - echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_DB_SUB_BLOCK}" "DB" "${db}" "${MODULE}"; + printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}"; include=0; if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then while read incl_db; do @@ -254,7 +257,7 @@ else fi; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUBEXT_BLOCK}" "PRUNE" "${db}" "$(date +'%F %T')" "${MODULE}"; echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; @@ -267,6 +270,8 @@ fi; if [ -z "${ONE_TIME_TAG}" ]; then # if this is borg version >1.2 we need to run compact after prune . "${DIR}/borg.backup.functions.compact.sh"; + # check in auto mode + . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.settings-default b/borg.backup.settings-default index f435336..9950f49 100644 --- a/borg.backup.settings-default +++ b/borg.backup.settings-default @@ -27,8 +27,13 @@ COMPRESSION_LEVEL=""; # Blank passwords allowed for only key (if used, use keyfile) # See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically ENCRYPTION=""; -# force repository check, default is off, set to true for check -FORCE_CHECK=""; +# force repository verirfy, default is off, set to true for verify on every run +FORCE_VERIFY=""; +# check interval, if 0 or negative number, no check will ever run +# if empty fall back to default set +# if set to 1 then every time the script runs +# any other value it means ever n days, eg 90 would be every 90 days +CHECK_INTERVAL=""; # default is %Y-%m-%d # todays date, if more than one per day add -%H%M for hour/minute # it can also be "{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f}" diff --git a/borg.backup.zabbix.sh b/borg.backup.zabbix.sh index 772c60b..2cb6baf 100755 --- a/borg.backup.zabbix.sh +++ b/borg.backup.zabbix.sh @@ -10,8 +10,11 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # init system . "${DIR}/borg.backup.functions.init.sh"; -# init verify file -BACKUP_INIT_VERIFY="borg.backup.zabbix.init"; +# init verify and check file +BACKUP_INIT_FILE="borg.backup.${MODULE}.init"; +BACKUP_CHECK_FILE="borg.backup.${MODULE}.check"; +# lock file +BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # verify valid data . "${DIR}/borg.backup.functions.verify.sh"; @@ -64,7 +67,7 @@ if [ -z "${BACKUP_SET_PREFIX}" ]; then BORG_PRUNE=$(echo "${BORG_PRUNE}" | sed -e 's/-P //'); fi; -echo "--- [BACKUP: zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; +printf "${PRINTF_SUB_BLOCK}" "BACKUP: zabbix settings" "$(date +'%F %T')" "${MODULE}"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}" if [ -z "${ONE_TIME_TAG}" ]; then @@ -75,10 +78,12 @@ if [ ${DRYRUN} -eq 0 ]; then ${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}; fi; if [ -z "${ONE_TIME_TAG}" ]; then - echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + printf "${PRINTF_SUB_BLOCK}" "PRUNE" "$(date +'%F %T')" "${MODULE}"; ${BORG_PRUNE}; # if this is borg version >1.2 we need to run compact after prune . "${DIR}/borg.backup.functions.compact.sh"; + # check in auto mode + . "${DIR}/borg.backup.functions.check.sh" "auto"; fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.mount.sh b/borg.mount.sh index 19e4c47..ef980d7 100755 --- a/borg.mount.sh +++ b/borg.mount.sh @@ -115,4 +115,4 @@ else borg umount "${MOUNT_PATH}"; fi; -## END +# __END__