diff --git a/Readme.md b/Readme.md index dd5ce7c..be4ff1a 100644 --- a/Readme.md +++ b/Readme.md @@ -4,6 +4,29 @@ These scripts are wrappers around the main borg backup scripts. Modules for plain file backup, mysql and postgresql backup exists. +## IMPORTANT NOTICE FOR UPGRADE TO VERSION 4.0 OR HIGHER + +*VERSION 4.0* CHANGE + +Version 4.0 introduces default borg repository name with `-file` for the `file` module. The repository has to be renamed manual before the next backup or the backup will fail. + +*Example:* + +Old backup name +```sh +BACKUP_FILE="some-backup-data.borg" +``` +Then the file need to be renamed the following way: +`mv some-backup-data.borg some-backup-data-file.borg` + +Below changes have to be done after the `file` module backup has been renamed. + +With Version 4.0 all backup sets are prefixed with the module name and a comma. For exmaple the files backup will have backup set "file,YYYY-MM-DD" as base name. + +To make sure prune of archives will work the `_borg_backup_set_prefix_cleanup.sh` script has to be run once. It has the same config (-c), debug (-d) and dry run (-n) options like the main scripts. It is recommended to run with the dry-run script first and see that the list of chagnes matches the expectation. + +The zabbix module has the prefix changed from `zabbix-settings-` to `zabbix,settings-` to match the new archive set rules + ## Recommended setup git clone this repostory into the target folder: @@ -16,6 +39,53 @@ Now the core scripts can be updated with a simple No settings files will be overwritten +## Possible command line options + +### `-c ` +if this is not given, /usr/local/scripts/borg/ is used + +### `-L ` +override config set and default log folder + +### `-T ` +create one time stand alone backup prefixed with tag name + +### `-D ` +remove a tagged backup set, full name must be given + +### `-b ` +override the default borg executable found in path + +### `-P` +print list of archives created + +### `-C` +check if repository exists, if not abort + +### `-E` +exit after check + +### `-I` +init repository (must be run first) + +### `-i` +print out only info + +### `-l` +list files during backup + +### `-v` +be verbose + +### `-d` +debug output all commands + +### `-n` +only do dry run + +### `-h` +this help page + ## Basic Settings `borg.backup.settings` @@ -60,7 +130,7 @@ All module settings files can have the following prefixed with `SUB_` to overrid * SUB_KEEP_YEARS * SUB_KEEP_WITHIN -## Setup backup via SSH to remote host +## Setup backup via SSH to remote host on `borg.backup.settings` For this the following settings are from interest @@ -74,8 +144,16 @@ Note that if `.ssh/config` is used only `TARGET_HOST` needs to be set. Recommene and `TARGET_BORG_PATH="";` if the target borg is in a non default path +## Override borg executable in `borg.backup.settings` + +`BORG_EXECUTABLE=""` + ## File backup settings +On new setups it is recommended to use the `borg.backup.file.setings` and set +`FILE_REPOSITORY_COMPATIBLE` +to `true` + ### Config variables diff --git a/_borg_backup_set_prefix_cleanup.sh b/_borg_backup_set_prefix_cleanup.sh new file mode 100755 index 0000000..ecbb94e --- /dev/null +++ b/_borg_backup_set_prefix_cleanup.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +# this will fix backup sets name +# must have a target call for +# file +# gitea +# mysql +# pgsql +# zabbix-settings- + +export BORG_BASE_DIR="borg/"; +DRYRUN=0; +TARGET_USER=""; +TARGET_HOST=""; +TARGET_PORT=""; +TARGET_BORG_PATH=""; +TARGET_FOLDER=""; +BASE_FOLDER="/usr/local/scripts/borg/"; +# those are the valid modules +MODULE_LIST="file gitea mysql pgsql zabbix" + +# basic options +# -c for config file override +# -n for dry run test +while getopts ":c:nd" opt; do + case "${opt}" in + c|config) + BASE_FOLDER=${OPTARG}; + ;; + d|debug) + DEBUG=1; + ;; + n|dryrun) + DRYRUN=1; + ;; + :) + echo "Option -$OPTARG requires an argument." + ;; + \?) + echo -e "\n Option does not exist: ${OPTARG}\n"; + usage; + exit 1; + ;; + esac; +done; + + +if [ ! -d "${BASE_FOLDER}" ]; then + echo "Base folder not found: ${BASE_FOLDER}"; + exit 1; +fi; +SETTINGS_FILE="borg.backup.settings"; + +. "${BASE_FOLDER}${SETTINGS_FILE}"; +ORIG_BACKUPFILE=${BACKUP_FILE}; +for MODULE in ${MODULE_LIST}; do + echo "************* MODULE: ${MODULE}"; + BACKUP_FILE=${ORIG_BACKUPFILE}; + # if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then + # . "${BASE_FOLDER}${SETTINGS_FILE_SUB}"; + # fi; + # SETTINGS_FILE_SUB=$(echo "${SETTINGS_FILE}" | sed -e "s/\.settings/\.${MODULE,,}\.settings/"); + if [ "${MODULE,,}" != "file" ]; then + BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg}; + fi; + TARGET_FOLDER=${TARGET_FOLDER%/} + TARGET_FOLDER=${TARGET_FOLDER#/} + # and add slash front and back and escape the path + TARGET_FOLDER=$(printf "%q" "/${TARGET_FOLDER}/"); + if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then + TARGET_SERVER="ssh://${TARGET_USER}@${TARGET_HOST}:${TARGET_PORT}/"; + # host/port + elif [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then + TARGET_SERVER="ssh://${TARGET_HOST}:${TARGET_PORT}/"; + # user/host + elif [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then + TARGET_SERVER="${TARGET_USER}@${TARGET_HOST}:"; + # host + elif [ ! -z "${TARGET_HOST}" ]; then + TARGET_SERVER="${TARGET_HOST}:"; + fi; + # we dont allow special characters, so we don't need to special escape it + REPOSITORY="${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}"; + echo "==== REPOSITORY: ${REPOSITORY}"; + borg list --format '{archive}{NL}' ${REPOSITORY}|grep -v "${MODULE},"| + while read i; do + # for gitea, zabbix we do not ADD we RENAME + if [ "${MODILE}" = "gitea" ]; then + target_name=$(echo $i | sed -e 's/gitea-/gitea,/'); + elif [ "${MODILE}" = "zabbix" ]; then + target_name=$(echo $i | sed -e 's/zabbix-settings-/zabbix,settings-/'); + else + target_name="${MODULE},${i}"; + fi; + echo "- Rename from: ${i} to: ${target_name}"; + if [ ${DEBUG} -eq 1 ]; then + echo "borg rename -p -v ${REPOSITORY}::${i} ${target_name}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + borg rename -p -v ${REPOSITORY}::${i} ${target_name}; + fi; + done; +done; + +# __END__ diff --git a/borg.backup.file.settings-default b/borg.backup.file.settings-default index c7096d8..aeeaaa3 100644 --- a/borg.backup.file.settings-default +++ b/borg.backup.file.settings-default @@ -2,8 +2,8 @@ # rename to borg.backup.file.settings to use -# set to false to use -file, current default is "true" -#FILE_REPOSITORY_COMPATIBLE="false" +# set to true to use old setting without -file in backup name +#FILE_REPOSITORY_COMPATIBLE="true" # override settings in borg.backup.settings with SUB_ prefix # valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_* diff --git a/borg.backup.file.sh b/borg.backup.file.sh index af26040..a3a6617 100755 --- a/borg.backup.file.sh +++ b/borg.backup.file.sh @@ -5,7 +5,7 @@ # set last edit date + time MODULE="file"; -MODULE_VERSION="1.0.0"; +MODULE_VERSION="1.2.0"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -22,6 +22,7 @@ BACKUP_INIT_CHECK="borg.backup.file.init"; # exit if include file is missing if [ ! -f "${BASE_FOLDER}${INCLUDE_FILE}" ]; then echo "[! $(date +'%F %T')] The include folder file ${INCLUDE_FILE} is missing"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; echo "--- [INCLUDE: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; @@ -146,11 +147,14 @@ if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then OPT_EXCLUDE="--exclude-from=${TMP_EXCLUDE_FILE}"; fi; fi; + +# set a special file prefix +BACKUP_SET_PREFIX="${MODULE},"; # add the repository set before we add the folders # base command -COMMAND="borg create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} "; +COMMAND="${BORG_COMMAND} create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} "; # add repoistory, after that the folders will be added on call -COMMAND=${COMMAND}${REPOSITORY}::${BACKUP_SET}; +COMMAND=${COMMAND}${REPOSITORY}::${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}; # if info print info and then abort run . "${DIR}/borg.backup.functions.info.sh"; @@ -172,19 +176,26 @@ if [ $FOLDER_OK -eq 1 ]; then fi; else echo "[! $(date +'%F %T')] No folders where set for the backup"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; -# clean up, always verbose -echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; -# build command -COMMAND="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} ${KEEP_OPTIONS[*]} ${REPOSITORY}"; -echo "Prune repository with keep${KEEP_INFO:1}"; -if [ ${DEBUG} -eq 1 ]; then - echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g'; +# clean up, always verbose, but only if we do not run one time tag +if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # build command + COMMAND="${BORG_COMMAND} prune ${OPT_REMOTE} -v --list ${OPT_PROGRESS} ${DRY_RUN_STATS} -P ${BACKUP_SET_PREFIX} ${KEEP_OPTIONS[*]} ${REPOSITORY}"; + echo "Prune repository with keep${KEEP_INFO:1}"; + if [ ${DEBUG} -eq 1 ]; then + echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g'; + fi; + # for the IFS="#" to work we need to replace options spaces with exactly ONE # + $(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted"; + # if this is borg version >1.2 we need to run compact after prune + . "${DIR}/borg.backup.functions.compact.sh"; +else + echo "[#] No prune with tagged backup"; fi; -# for the IFS="#" to work we need to replace options spaces with exactly ONE # -$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted"; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.functions.check.sh b/borg.backup.functions.check.sh index 5848496..62a30de 100644 --- a/borg.backup.functions.check.sh +++ b/borg.backup.functions.check.sh @@ -10,6 +10,8 @@ echo "Script version: ${VERSION}"; # show type echo "Backup module : ${MODULE}"; echo "Module version: ${MODULE_VERSION}"; +# borg version +echo "Borg version : ${BORG_VERSION}"; # show base folder always echo "Base folder : ${BASE_FOLDER}"; @@ -130,63 +132,71 @@ if [ ! -w "${HOME}" ] || [ "${HOME}" = '/' ]; then HOME=$(eval echo "$(whoami)"); fi; -# build options and info string, -# also flag BACKUP_SET check if hourly is set +# keep optionfs (for files) KEEP_OPTIONS=(); +# keep info string (for files) KEEP_INFO=""; -BACKUP_SET_CHECK=0; -if [ ${KEEP_LAST} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}"); - KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}"; -fi; -if [ ${KEEP_HOURS} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}"); - KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}"; - BACKUP_SET_CHECK=1; -fi; -if [ ${KEEP_DAYS} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}"); - KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}"; -fi; -if [ ${KEEP_WEEKS} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}"); - KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}"; -fi; -if [ ${KEEP_MONTHS} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}"); - KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}"; -fi; -if [ ${KEEP_YEARS} -gt 0 ]; then - KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}"); - KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}"; -fi; -if [ ! -z "${KEEP_WITHIN}" ]; then - # check for invalid string. can only be number + H|d|w|m|y - if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then - KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}"); - KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}"; - if [[ "${KEEP_WITHIN}" == *"H"* ]]; then - BACKUP_SET_CHECK=1; +# override standard keep for tagged backups +if [ ! -z "${ONE_TIME_TAG}" ]; then + BACKUP_SET="{now:%Y-%m-%dT%H:%M:%S}"; +else + # build options and info string, + # also flag BACKUP_SET check if hourly is set + BACKUP_SET_CHECK=0; + if [ ${KEEP_LAST} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}"); + KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}"; + fi; + if [ ${KEEP_HOURS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}"); + KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}"; + BACKUP_SET_CHECK=1; + fi; + if [ ${KEEP_DAYS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}"); + KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}"; + fi; + if [ ${KEEP_WEEKS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}"); + KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}"; + fi; + if [ ${KEEP_MONTHS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}"); + KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}"; + fi; + if [ ${KEEP_YEARS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}"); + KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}"; + fi; + if [ ! -z "${KEEP_WITHIN}" ]; then + # check for invalid string. can only be number + H|d|w|m|y + if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then + KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}"); + KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}"; + if [[ "${KEEP_WITHIN}" == *"H"* ]]; then + BACKUP_SET_CHECK=1; + fi; + else + echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string."; + exit 1; fi; - else - echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string."; + fi; + # abort if KEEP_OPTIONS is empty + if [ -z "${KEEP_OPTIONS}" ]; then + echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format."; exit 1; fi; + # set BACKUP_SET if empty, set to Year-month-day + if [ -z "${BACKUP_SET}" ]; then + BACKUP_SET="{now:%Y-%m-%d}"; + fi; + # backup set check, and there is no hour entry (%H) in the archive string + # we add T%H:%M:%S in this case, before the last } + if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then + BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/"); + fi; fi; -# abort if KEEP_OPTIONS is empty -if [ -z "${KEEP_OPTIONS}" ]; then - echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format."; - exit 1; -fi; -# set BACKUP_SET if empty, set to Year-month-day -if [ -z "${BACKUP_SET}" ]; then - BACKUP_SET="{now:%Y-%m-%d}"; -fi; -# backup set check, and there is no hour entry (%H) in the archive string -# we add T%H:%M:%S in this case, before the last } -if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then - BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/"); -fi; + # for folders list split set to "#" and keep the old setting as is _IFS=${IFS}; @@ -198,8 +208,8 @@ fi; # borg call, replace ##...## parts during run # used in all modules, except 'file' -_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; -_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; +_BORG_CALL="${BORG_COMMAND} create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; +_BORG_PRUNE="${BORG_COMMAND} prune ${OPT_REMOTE} -v --list ${OPT_PROGRESS} ${DRY_RUN_STATS} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; # general borg settings # set base path to config directory to keep cache/config separated @@ -216,7 +226,7 @@ if [ ${DEBUG} -eq 1 ]; then fi; # prepare debug commands only COMMAND_EXPORT="export BORG_BASE_DIR=\"${BASE_FOLDER}\";" -COMMAND_INFO="${COMMAND_EXPORT}borg info ${OPT_REMOTE} ${REPOSITORY}"; +COMMAND_INFO="${COMMAND_EXPORT}${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}"; # if the is not there, call init to create it # if this is user@host, we need to use ssh command to check if the file is there # else a normal check is ok @@ -225,14 +235,13 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then echo "--- [CHECK : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; if [ ! -z "${TARGET_SERVER}" ]; then if [ ${DEBUG} -eq 1 ]; then - echo "borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\""; + echo "${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\""; fi; # use borg info and check if it returns "Repository ID:" in the first line - REPO_CHECK=$(borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:"); + REPO_CHECK=$(${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:"); # this is currently a hack to work round the error code in borg info # this checks if REPO_CHECK holds this error message and then starts init - regex="^Some part of the script failed with an error:"; - if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${regex} ]]; then + if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${REGEX_ERROR} ]]; then INIT_REPOSITORY=1; fi; elif [ ! -d "${REPOSITORY}" ]; then @@ -248,37 +257,39 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then # end if checked but repository is not here if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then echo "[! $(date +'%F %T')] No repository. Please run with -I flag to initialze repository"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ${EXIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ]; then echo "Repository exists"; echo "For more information run:" echo "${COMMAND_INFO}"; - echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + . "${DIR}/borg.backup.functions.close.sh"; exit; fi; fi; if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then echo "--- [INIT : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then - echo "borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}"; + echo "${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}"; fi if [ ${DRYRUN} -eq 0 ]; then # should trap and exit properly here - borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}; + ${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}; # write init file echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}"; echo "Repository initialized"; echo "For more information run:" echo "${COMMAND_INFO}"; fi - echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + . "${DIR}/borg.backup.functions.close.sh"; # exit after init exit; elif [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 0 ]; then echo "[! $(date +'%F %T')] Repository already initialized"; echo "For more information run:" echo "${COMMAND_INFO}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; @@ -286,6 +297,7 @@ fi; if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then echo "[! $(date +'%F %T')] It seems the repository has never been initialized." echo "Please run -I to initialize or if already initialzed run with -C for init update." + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; @@ -295,7 +307,7 @@ if [ ${PRINT} -eq 1 ]; then FORMAT="{archive} {comment:6} {start} - {end} [{id}] ({username}@{hostname}){NL}" # show command on debug or dry run if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}"; fi; # run info command if not a dry drun if [ ${DRYRUN} -eq 0 ]; then @@ -304,17 +316,60 @@ if [ ${PRINT} -eq 1 ]; then if [ ${VERBOSE} -eq 1 ]; then echo ""; echo "Base command info:" - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]"; echo "Replace [COMMAND] with list for listing or extract for restoring backup data." echo "Replace [BACKUP] with archive name." echo "If no [PATH] is given then all files will be restored." echo "Before extracting -n (dry run) is recommended to use." echo "If archive size is needed the info command with archive name has to be used." - echo "When listing (list) data the --format command can be used." + echo "When listing files in an archive set (::SET) the --format command can be used." echo "Example: \"{mode} {user:6} {group:6} {size:8d} {csize:8d} {dsize:8d} {dcsize:8d} {mtime} {path}{extra} [{health}]{NL}\"" else - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]"; fi; + . "${DIR}/borg.backup.functions.close.sh"; + exit; +fi; + +# DELETE ONE TIME TAG +if [ ! -z "${DELETE_ONE_TIME_TAG}" ]; then + echo "--- [DELETE: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # if a "*" is inside we don't do ONE archive, but globbing via -a option + DELETE_ARCHIVE="" + OPT_GLOB=""; + # this is more or less for debug only + if [[ "${DELETE_ONE_TIME_TAG}" =~ $REGEX_GLOB ]]; then + OPT_GLOB="-a '${DELETE_ONE_TIME_TAG}'" + else + DELETE_ARCHIVE="::"${DELETE_ONE_TIME_TAG}; + fi + # if this is borg <1.2 OPT_LIST does not work + if [ $(version $BORG_VERSION) -lt $(version "1.2.0") ]; then + OPT_LIST=""; + fi; + # if exists, delete and exit + # show command on debug or dry run + if [ ${DEBUG} -eq 1 ]; then + echo "${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} -s ${OPT_GLOB} ${REPOSITORY}${DELETE_ARCHIVE}"; + fi; + # run delete command if not a dry drun + # NOTE seems to be glob is not working if wrapped into another variable + if [[ "${DELETE_ONE_TIME_TAG}" =~ $REGEX_GLOB ]]; then + ${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} ${DRY_RUN_STATS} -a "${DELETE_ONE_TIME_TAG}" ${REPOSITORY}; + else + ${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} ${DRY_RUN_STATS} ${REPOSITORY}${DELETE_ARCHIVE}; + fi; + # if not a dry run, compact repository after delete + # not that compact only works on borg 1.2 + if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then + if [ ${DRYRUN} -eq 0 ]; then + ${BORG_COMMAND} compact ${REPOSITORY}; + fi; + if [ ${DEBUG} -eq 1 ]; then + echo "${BORG_COMMAND} compact ${REPOSITORY}"; + fi; + fi; + . "${DIR}/borg.backup.functions.close.sh"; exit; fi; diff --git a/borg.backup.functions.close.sh b/borg.backup.functions.close.sh index 07e1d79..877098b 100644 --- a/borg.backup.functions.close.sh +++ b/borg.backup.functions.close.sh @@ -2,8 +2,13 @@ # unset borg settings unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK -DURATION=$[ $(date +'%s')-$START ]; -echo "=== [Run time: $(convert_time ${DURATION})]"; -echo "=== [END : $(date +'%F %T')] ==[${MODULE}]===================>"; +# error abort without duration and error notice +if [ $1 == 1 ]; then + echo "=== [ERROR: $(date +'%F %T')] ==[${MODULE}]====================================>"; +else + DURATION=$[ $(date +'%s')-$START ]; + echo "=== [Run time: $(convert_time ${DURATION})]"; + echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; +fi; # __END__ diff --git a/borg.backup.functions.compact.sh b/borg.backup.functions.compact.sh new file mode 100644 index 0000000..ca2ff3c --- /dev/null +++ b/borg.backup.functions.compact.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# compact (only if BORG COMPACT is set) +# only for borg 1.2 + +# reset to normal IFS, so command works here +IFS=${_IFS}; +if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then + echo "--- [COMPACT:$(date +'%F %T')] --[${MODULE}]------------------------------------>"; + BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}"; + if [ ${DEBUG} -eq 1 ]; then + echo "${BORG_COMPACT}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + ${BORG_COMPACT}; + fi; +fi; + +# __END__ diff --git a/borg.backup.functions.info.sh b/borg.backup.functions.info.sh index b283c6f..2b0ebdc 100644 --- a/borg.backup.functions.info.sh +++ b/borg.backup.functions.info.sh @@ -4,11 +4,11 @@ if [ ${INFO} -eq 1 ]; then echo "--- [INFO : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; # show command on debug or dry run if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then - echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg info ${OPT_REMOTE} ${REPOSITORY}"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}"; fi; # run info command if not a dry drun if [ ${DRYRUN} -eq 0 ]; then - borg info ${OPT_REMOTE} ${REPOSITORY}; + ${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}; fi; if [ "${MODULE}" = "files" ]; then if [ $FOLDER_OK -eq 1 ]; then @@ -23,7 +23,7 @@ if [ ${INFO} -eq 1 ]; then rm -f "${TMP_EXCLUDE_FILE}"; fi; fi; - echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + . "${DIR}/borg.backup.functions.close.sh"; exit; fi; diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index b03bea6..261b3d1 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -13,10 +13,17 @@ cleanup() { } # on exit unset any exported var trap "unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK" EXIT; +# for version compare +function version { + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; +} # version for all general files -VERSION="3.0.2"; +VERSION="4.1.0"; +# borg version and borg comamnd +BORG_VERSION=""; +BORG_COMMAND="borg"; # default log folder if none are set in config or option _LOG_FOLDER="/var/log/borg.backup/"; # log file name is set based on BACKUP_FILE, .log is added @@ -34,6 +41,9 @@ INCLUDE_FILE=""; EXCLUDE_FILE=""; # backup folder initialzed check BACKUP_INIT_CHECK=""; +# one time backup prefix tag, if set will use .-Y-M-DTh:m:s type backup prefix +ONE_TIME_TAG=""; +DELETE_ONE_TIME_TAG=""; # debug/verbose VERBOSE=0; LIST=0; @@ -51,13 +61,14 @@ _BORG_RELOCATED_REPO_ACCESS_IS_OK="yes"; # NOTE: to keep the old .borg repository name for file module set this to true # if set to false (future) it will add -file to the repository name like for other # modules -FILE_REPOSITORY_COMPATIBLE="true"; +FILE_REPOSITORY_COMPATIBLE="false"; # other variables TARGET_SERVER=""; REGEX=""; REGEX_COMMENT="^[\ \t]*#"; REGEX_GLOB='\*'; REGEX_NUMERIC="^[0-9]{1,2}$"; +REGEX_ERROR="^Some part of the script failed with an error:"; PRUNE_DEBUG=""; INIT_REPOSITORY=0; FOLDER_OK=0; @@ -77,6 +88,10 @@ TARGET_BORG_PATH=""; TARGET_FOLDER=""; BACKUP_FILE=""; SUB_BACKUP_FILE=""; +# OPT is for options set +OPT_BORG_EXECUTEABLE=""; +# which overrides BORG_EXECUTABLE that can be set in the settings file +BORG_EXECUTEABLE=""; # lz4, zstd 1-22 (3), zlib 0-9 (6), lzma 0-9 (6) DEFAULT_COMPRESSION="zstd"; DEFAULT_COMPRESSION_LEVEL=3; @@ -147,13 +162,16 @@ function usage() -c : if this is not given, ${BASE_FOLDER} is used -L : override config set and default log folder + -T : create one time stand alone backup prefixed with tag name + -D : remove a tagged backup set, full name must be given + -b : override default path -P: print list of archives created -C: check if repository exists, if not abort -E: exit after check -I: init repository (must be run first) - -v: be verbose -i: print out only info -l: list files during backup + -v: be verbose -d: debug output all commands -n: only do dry run -h: this help page @@ -165,14 +183,23 @@ function usage() } # set options -while getopts ":c:L:vldniCEIPh" opt; do +while getopts ":c:L:T:D:b:vldniCEIPh" opt; do case "${opt}" in c|config) BASE_FOLDER=${OPTARG}; ;; - L|log) + L|Log) OPT_LOG_FOLDER=${OPTARG}; ;; + T|Tag) + ONE_TIME_TAG=${OPTARG}; + ;; + D|Delete) + DELETE_ONE_TIME_TAG=${OPTARG}; + ;; + b|borg) + OPT_BORG_EXECUTEABLE=${OPTARG}; + ;; C|Check) # will check if repo is there and abort if not CHECK=1; @@ -244,10 +271,36 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${INFO} -eq 1 ]; then exit 1; fi; # print -P cannot be run with -i/-C/-I together -if [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INFO} -eq 1 ]; then +if [ ${PRINT} -eq 1 ] && ([ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then echo "Cannot have -P print option and -i info, -C check or -I initizalize option at the same time"; exit 1; fi; +# if tag is set, you can't have init, check, info, etc +if [ ! -z "${ONE_TIME_TAG}" ] && ([ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then + echo "Cannot have -T '${ONE_TIME_TAG}' option with -i info, -C check, -I initialize or -P print option at the same time"; + exit 1; +fi; +# check only alphanumeric, no spaces, only underscore and dash +if [ ! -z "${ONE_TIME_TAG}" ] && ! [[ "${ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "One time tag '${ONE_TIME_TAG}' must be alphanumeric with dashes and underscore only."; + exit 1; +elif [ ! -z "${ONE_TIME_TAG}" ]; then + # all ok, attach . at the end + ONE_TIME_TAG=${ONE_TIME_TAG}"."; +fi; +# if -D, cannot be with -T, -i, -C, -I, -P +if [ ! -z "${DELETE_ONE_TIME_TAG}" ] && ([ ! -z "${ONE_TIME_TAG}" ] || [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then + echo "Cannot have -D delete tag option with -T one time tag, -i info, -C check, -I initialize or -P print option at the same time"; + exit 1; +fi; +# -D also must be in valid backup set format +# ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]] +if [ ! -z "${DELETE_ONE_TIME_TAG}" ] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},([A-Za-z0-9_-]+-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$ ]] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]]; then + echo "Delete one time tag '${DELETE_ONE_TIME_TAG}' is in an invalid format. Please check existing tags with -P option." + echo "For a globing be sure it is in the format of: TAG.MODULE,*-YYYY-MM-DDT*"; + echo "Note the dash (-) after the first *, also time (T) is a globa (*) must." + exit 1; +fi; # verbose & progress if [ ${VERBOSE} -eq 1 ]; then @@ -258,12 +311,45 @@ fi; if [ ${LIST} -eq 1 ]; then OPT_LIST="--list"; fi; +# If dry run, the stats (-s) option cannot be used if [ ${DRYRUN} -eq 1 ]; then - PRUNE_DEBUG="--dry-run"; + DRY_RUN_STATS="-n"; +else + DRY_RUN_STATS="-s"; fi; # read config file . "${BASE_FOLDER}${SETTINGS_FILE}"; + +# if OPTION SET overrides ALL others +if [ ! -z "${OPT_BORG_EXECUTEABLE}" ]; then + BORG_COMMAND="${OPT_BORG_EXECUTEABLE}"; + if [ ! -f "${BORG_COMMAND}" ]; then + echo "borg command not found with option -b: ${BORG_COMMAND}"; + exit; + fi; +# if in setting file, use this +elif [ ! -z "${BORG_EXECUTEABLE}" ]; then + BORG_COMMAND="${BORG_EXECUTEABLE}"; + if [ ! -f "${BORG_COMMAND}" ]; then + echo "borg command not found with setting: ${BORG_COMMAND}"; + exit; + fi; +elif ! command -v borg &> /dev/null; then + echo "borg backup seems not to be installed, please check paths"; + exit; +fi; +# check that this is a borg executable, no detail check +_BORG_COMMAND_CHECK=$(${BORG_COMMAND} -V | grep "borg"); +if [[ "${_BORG_COMMAND_CHECK}" =~ ${REGEX_ERROR} ]]; then + echo "Cannot extract borg info from command, is this a valid borg executable?: ${BORG_COMMAND}"; + exit; +fi; +# extract actually borg version from here +# alt sed to get only numbes: sed -e 's/.* \([0-9]*\.[0-9]*\.[0-9]*\)$/\1/g' +# or use cut -d " " -f 2 and assume NO space in the first part +BORG_VERSION=$(${BORG_COMMAND} -V | sed -e 's/borg.* //') 2>&1 || echo "[!] Borg version not estable"; + # load default settings for fileds not set if [ -z "${COMPRESSION}" ]; then COMPRESSION="${DEFAULT_COMPRESSION}"; @@ -341,7 +427,7 @@ if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then fi; # add module name to backup file, always # except if FILE module and FILE_REPOSITORY_COMPATIBLE="true" -if ([ "${FILE_REPOSITORY_COMPATIBLE}" = "false" ] && [ "${MODULE,,}" = "file" ]) || [ "${MODULE,,}" != "file" ]; then +if [ "${FILE_REPOSITORY_COMPATIBLE}" != "true" ] || [ "${MODULE,,}" != "file" ]; then BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg}; fi; # backup file must be set diff --git a/borg.backup.gitea.sh b/borg.backup.gitea.sh index 960bf52..41c31fd 100755 --- a/borg.backup.gitea.sh +++ b/borg.backup.gitea.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash MODULE="gitea" -MODULE_VERSION="1.0.0"; +MODULE_VERSION="1.1.1"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -35,17 +35,19 @@ if [ -z "${GITEA_CONFIG}" ]; then fi; if [ ! -f "${GITEA_BIN}" ]; then echo "[! $(date +'%F %T')] Cannot find gitea binary"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ! -f "${GITEA_CONFIG}" ]; then echo "[! $(date +'%F %T')] Cannot find gitea config"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; # Filename FILENAME="gitea.backup.zip"; # backup set and prefix -BACKUP_SET_PREFIX="gitea-"; -BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${BACKUP_SET}"; +BACKUP_SET_PREFIX="${MODULE},"; +BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); @@ -53,7 +55,9 @@ BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SE echo "--- [git data and database: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}"; - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then ( @@ -69,8 +73,13 @@ if [ ${DRYRUN} -eq 0 ]; then sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}; ) | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' # remove all ESC strings fi; -echo "Prune repository with keep${KEEP_INFO:1}"; -${BORG_PRUNE}; +if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + # if this is borg version >1.2 we need to run compact after prune + . "${DIR}/borg.backup.functions.compact.sh"; +fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh index 3a184d0..4dd1ac1 100755 --- a/borg.backup.mysql.sh +++ b/borg.backup.mysql.sh @@ -10,7 +10,7 @@ # set last edit date + time MODULE="mysql" -MODULE_VERSION="1.0.0"; +MODULE_VERSION="1.1.0"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -42,10 +42,12 @@ MYSQL_CMD=${MYSQL_BASE_PATH}'mysql'; # no dump or mysql, bail if [ ! -f "${MYSQL_DUMP}" ]; then echo "[! $(date +'%F %T')] mysqldump binary not found"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ! -f "${MYSQL_CMD}" ]; then echo "[! $(date +'%F %T')] mysql binary not found"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; # check that the user can actually do, else abort here @@ -54,6 +56,7 @@ _MYSQL_CHECK=$(mysqladmin ${MYSQL_DB_CONFIG_PARAM} ping 2>&1); _MYSQL_OK=$(echo "${_MYSQL_CHECK}" | grep "is alive"); if [ -z "${_MYSQL_OK}" ]; then echo "[! $(date +'%F %T')] Current user has no access right to mysql database"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; # below is for file name only @@ -89,35 +92,42 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='--no-data'; schema_flag='schema'; fi; - echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "---[BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; # We only do a full backup and not per table backup here # Filename FILENAME="all-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql" # backup set: - BACKUP_SET_PREFIX="all-"; - BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${MODULE},all-"; + BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}"; - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then ${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; fi; - echo "Prune repository with keep${KEEP_INFO:1}"; - ${BORG_PRUNE}; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + fi; else ${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" | while read db; do - echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "========[DB: ${db}]========================[${MODULE}]====================================>"; + echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; # exclude checks include=0; if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then @@ -174,8 +184,8 @@ else # prepare borg calls FILENAME="${db}-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql" # backup set: - BACKUP_SET_PREFIX="${db}-" - BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${MODULE},${db}-" + BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); @@ -190,16 +200,25 @@ else _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; fi; - echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; - ${BORG_PRUNE}; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + fi; else echo "- [E] ${db}"; fi; done; fi; +# run compact at the end if not a dry run +if [ -z "${ONE_TIME_TAG}" ]; then + # if this is borg version >1.2 we need to run compact after prune + . "${DIR}/borg.backup.functions.compact.sh"; +fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index 0f75fcb..cfc66bd 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -10,7 +10,7 @@ # set last edit date + time MODULE="pgsql" -MODULE_VERSION="1.0.1"; +MODULE_VERSION="1.1.1"; DIR="${BASH_SOURCE%/*}" @@ -43,6 +43,7 @@ _PATH_PG_VERSION=${PG_VERSION}; _backup_error=$?; if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then echo "[! $(date +'%F %T')] Cannot get PostgreSQL server version: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; @@ -58,6 +59,7 @@ if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then _PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//'); if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then echo "[! $(date +'%F %T')] PostgreSQL not found in any paths"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; fi; @@ -69,14 +71,17 @@ PG_DUMPALL=${PG_PATH}'pg_dumpall'; # check that command are here if [ ! -f "${PG_PSQL}" ]; then echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ! -f "${PG_DUMP}" ]; then echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ! -f "${PG_DUMPALL}" ]; then echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; @@ -95,58 +100,70 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='-s'; schema_flag='schema'; fi; - echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "--- [BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; # Filename FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" # backup set: - BACKUP_SET_PREFIX="all-"; - BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${MODULE},all-"; + BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}"; - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then $(${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}); _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; fi; - echo "Prune repository with keep${KEEP_INFO:1}"; - ${BORG_PRUNE}; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + fi; else # dump globals first db="pg_globals"; schema_flag="data"; - echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; # Filename FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" # backup set: - BACKUP_SET_PREFIX="${db}-"; - BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${MODULE},${db}-"; + BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}"; - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then ${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; fi; - echo "Prune repository with keep${KEEP_INFO:1}"; - ${BORG_PRUNE}; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + fi; # get list of tables for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do @@ -154,7 +171,8 @@ else owner=$(echo ${owner_db} | cut -d "," -f 1); db=$(echo ${owner_db} | cut -d "," -f 2); encoding=$(echo ${owner_db} | cut -d "," -f 3); - echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "========[DB: ${db}]========================[${MODULE}]====================================>"; + echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; include=0; if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then while read incl_db; do @@ -192,10 +210,10 @@ else # Filename # Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" - # backup set: - BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}"; # PER db either data or schema - BACKUP_SET_PREFIX="${db}-"; + BACKUP_SET_PREFIX="${MODULE},${db}-"; + # backup set: + BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); # borg prune @@ -203,23 +221,34 @@ else if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}"; - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then ${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit $_backup_error; fi; fi; - echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; - ${BORG_PRUNE}; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + fi; else echo "- [E] ${db}"; fi; done; fi; +# run compact at the end if not a dry run +if [ -z "${ONE_TIME_TAG}" ]; then + # if this is borg version >1.2 we need to run compact after prune + . "${DIR}/borg.backup.functions.compact.sh"; +fi; . "${DIR}/borg.backup.functions.close.sh"; diff --git a/borg.backup.zabbix.sh b/borg.backup.zabbix.sh index 0fdd4dd..af361cf 100755 --- a/borg.backup.zabbix.sh +++ b/borg.backup.zabbix.sh @@ -3,7 +3,7 @@ # Backup zabbix config and settings only MODULE="zabbix" -MODULE_VERSION="1.0.1"; +MODULE_VERSION="1.1.1"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -24,7 +24,8 @@ if [ -z "${ZABBIX_DUMP_BIN}" ]; then fi; if [ ! -z "${ZABBIX_CONFIG}" ] && [ ! -f "${ZABBIX_CONFIG}" ]; then echo "[! $(date +'%F %T')] Cannot find zabbix config: ${ZABBIX_CONFIG}"; - exit; + . "${DIR}/borg.backup.functions.close.sh" 1; + exit 1; fi; if [ -f "${ZABBIX_CONFIG}" ]; then OPT_ZABBIX_CONFIG="-z ${ZABBIX_CONFIG}"; @@ -34,10 +35,12 @@ if [ "${ZABBIX_DATABASE}" = "psql" ]; then fi; if [ "${ZABBIX_DATABASE}" != "psql" ] && [ "${ZABBIX_DATABASE}" != "mysql" ]; then echo "[! $(date +'%F %T')] Zabbix dump must have database set to either psql or mysql"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; if [ ! -f "${ZABBIX_DUMP_BIN}" ]; then echo "[! $(date +'%F %T')] Zabbix dump script could not be found: ${ZABBIX_DUMP_BIN}"; + . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; # -i (ignore)/ -f (backup) @@ -50,8 +53,8 @@ fi; # Filename FILENAME="zabbix-config.c.sql"; # backup set: -BACKUP_SET_PREFIX="zabbix-settings-"; -BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${BACKUP_SET}"; +BACKUP_SET_PREFIX="${MODULE},settings-"; +BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); @@ -61,13 +64,23 @@ if [ -z "${BACKUP_SET_PREFIX}" ]; then BORG_PRUNE=$(echo "${BORG_PRUNE}" | sed -e 's/-P //'); fi; -echo "--- [zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; +echo "--- [BACKUP: zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}" - echo "${BORG_PRUNE}"; + if [ -z "${ONE_TIME_TAG}" ]; then + echo "${BORG_PRUNE}"; + fi; fi; if [ ${DRYRUN} -eq 0 ]; then ${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}; fi; -echo "Prune repository with keep${KEEP_INFO:1}"; -${BORG_PRUNE}; +if [ -z "${ONE_TIME_TAG}" ]; then + echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + ${BORG_PRUNE}; + # if this is borg version >1.2 we need to run compact after prune + . "${DIR}/borg.backup.functions.compact.sh"; +fi; + +. "${DIR}/borg.backup.functions.close.sh"; + +# __EMD__