commit 93f8ea6054425d55db832d94e38303fb0ad2de6b Author: Clemens Schwaighofer Date: Mon Dec 13 10:36:44 2021 +0900 Borg Backup Wrapper Scripts diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..6c8b6f3 --- /dev/null +++ b/Readme.md @@ -0,0 +1,13 @@ +# Borg Backup Wrapper Scripts + +These scripts are wrappers around the main borg backup scripts. + +Modules for plain file backup, mysql and postgresql backup exists. + +## Basic Settings + +## File backup settings + +## PostgreSQL backup settings + +## MySQL backup settings diff --git a/borg.backup.file.sh b/borg.backup.file.sh new file mode 100755 index 0000000..7f4e103 --- /dev/null +++ b/borg.backup.file.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +# Run -I first to initialize repository +# There are no automatic repository checks unless -C is given + +# set last edit date + time +MODULE="file"; +MODULE_VERSION="0.1.0"; + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +. "${DIR}/borg.backup.functions.init.sh"; + +# include and exclude file +INCLUDE_FILE="borg.backup.file.include"; +EXCLUDE_FILE="borg.backup.file.exclude"; +BACKUP_INIT_CHECK="borg.backup.file.init"; + +. "${DIR}/borg.backup.functions.check.sh"; + +# exit if include file is missing +if [ ! -f "${BASE_FOLDER}${INCLUDE_FILE}" ]; then + echo "[! $(date +'%F %T')] The include folder file ${INCLUDE_FILE} is missing"; + exit 1; +fi; +# folders to backup +FOLDERS=(); +# this if for debug output with quoted folders +FOLDERS_Q=(); +# include list +while read include_folder; do + # strip any leading spaces from that folder + include_folder=$(echo "${include_folder}" | sed -e 's/^[ \t]*//'); + # check that those folders exist, warn on error, + # but do not exit unless there are no valid folders at all + # skip folders that are with # in front (comment) + if [[ "${include_folder}" =~ ${REGEX_COMMENT} ]]; then + echo "# [C] Comment: '${include_folder}'"; + else + # skip if it is empty + if [ ! -z "${include_folder}" ]; then + # if this is a glob, do a double check that the base folder actually exists (?) + if [[ "${include_folder}" =~ $REGEX_GLOB ]]; then + # if this is */ then allow it + # remove last element beyond the last / + # if there is no path, just allow it (general rule) + _include_folder=${include_folder%/*}; + # if still a * inside -> add as is, else check for folder + if [[ "${include_folder}" =~ $REGEX_GLOB ]]; then + FOLDER_OK=1; + echo "+ [I] Backup folder with folder path glob '${include_folder}'"; + # glob (*) would be escape so we replace it with a temp part and then reinsert it + FOLDERS_Q+=($(printf "%q" "$(echo "${include_folder}" | sed 's/\*/_STARGLOB_/g')" | sed 's/_STARGLOB_/\*/g')); + FOLDERS+=("${include_folder}"); + elif [ ! -d "${_include_folder}" ]; then + echo "- [I] Backup folder with glob '${include_folder}' does not exist or is not accessable"; + else + FOLDER_OK=1; + echo "+ [I] Backup folder with glob '${include_folder}'"; + # we need glob fix + FOLDERS_Q+=($(printf "%q" "$(echo "${include_folder}" | sed 's/\*/_STARGLOB_/g')" | sed 's/_STARGLOB_/\*/g')); + FOLDERS+=("${include_folder}"); + fi; + # normal folder + elif [ ! -d "${include_folder}" ] && [ ! -e "${include_folder}" ]; then + echo "- [I] Backup folder or file '${include_folder}' does not exist or is not accessable"; + else + FOLDER_OK=1; + # if it is a folder, remove the last / or the symlink check will not work + if [ -d "${include_folder}" ]; then + _include_folder=${include_folder%/*}; + else + _include_folder=${include_folder}; + fi; + # Warn if symlink & folder -> only smylink will be backed up + if [ -h "${_include_folder}" ]; then + echo "~ [I] Target '${include_folder}' is a symbolic link. No real data will be backed up"; + else + echo "+ [I] Backup folder or file '${include_folder}'"; + fi; + FOLDERS_Q+=($(printf "%q" "${include_folder}")); + FOLDERS+=("${include_folder}"); + fi; + fi; + fi; +done<"${BASE_FOLDER}${INCLUDE_FILE}"; + +# exclude list +if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then + # check that the folders in that exclude file are actually valid, + # remove non valid ones and warn + #TMP_EXCLUDE_FILE=$(mktemp --tmpdir ${EXCLUDE_FILE}.XXXXXXXX); # non mac + TMP_EXCLUDE_FILE=$(mktemp "${TEMPDIR}${EXCLUDE_FILE}".XXXXXXXX); + while read exclude_folder; do + # strip any leading spaces from that folder + exclude_folder=$(echo "${exclude_folder}" | sed -e 's/^[ \t]*//'); + # folder or any type of file is ok + # because of glob files etc, exclude only comments (# start) + if [[ "${exclude_folder}" =~ ${REGEX_COMMENT} ]]; then + echo "# [C] Comment: '${exclude_folder}'"; + else + # skip if it is empty + if [ ! -z "${exclude_folder}" ]; then + # if it DOES NOT start with a / we assume free folder and add as is + if [[ "${exclude_folder}" != /* ]]; then + echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE}; + echo "+ [E] General exclude: '${exclude_folder}'"; + # if this is a glob, do a double check that the base folder actually exists (?) + elif [[ "${exclude_folder}" =~ $REGEX_GLOB ]]; then + # remove last element beyond the last / + # if there is no path, just allow it (general rule) + _exclude_folder=${exclude_folder%/*}; + if [ ! -d "${_exclude_folder}" ]; then + echo "- [E] Exclude folder with glob '${exclude_folder}' does not exist or is not accessable"; + else + echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE}; + echo "+ [E] Exclude folder with glob '${exclude_folder}'"; + fi; + # do a warning for a possible invalid folder + # but we do not a exclude if the data does not exist + elif [ ! -d "${exclude_folder}" ] && [ ! -e "${exclude_folder}" ]; then + echo "- [E] Exclude folder or file '${exclude_folder}' does not exist or is not accessable"; + else + echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE}; + # if it is a folder, remove the last / or the symlink check will not work + if [ -d "${exclude_folder}" ]; then + _exclude_folder=${exclude_folder%/*}; + else + _exclude_folder=${exclude_folder}; + fi; + # warn if target is symlink folder + if [ -h "${_exclude_folder}" ]; then + echo "~ [I] Target '${exclude_folder}' is a symbolic link. No real data will be excluded from backup"; + else + echo "+ [E] Exclude folder or file '${exclude_folder}'"; + fi; + fi; + fi; + fi; + done<"${BASE_FOLDER}${EXCLUDE_FILE}"; + # avoid blank file add by checking if the tmp file has a size >0 + if [ -s "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then + OPT_EXCLUDE="--exclude-from=${TMP_EXCLUDE_FILE}"; + fi; +fi; +# add the repository set before we add the folders +# base command +COMMAND="borg create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} "; +# add repoistory, after that the folders will be added on call +COMMAND=${COMMAND}${REPOSITORY}::${BACKUP_SET}; +# if info print info and then abort run +. "${DIR}/borg.backup.functions.info.sh"; + +if [ $FOLDER_OK -eq 1 ]; then + echo "--- [BACKUP: $(date +'%F %T')] ------------------------------------------->"; + # show command + if [ ${DEBUG} -eq 1 ]; then + echo $(echo ${COMMAND} | sed -e 's/[ ][ ]*/ /g') ${FOLDERS_Q[*]}; + fi; + # execute backup command + if [ ${DRYRUN} -eq 0 ]; then + # need to redirect std error to std out so all data is printed to the correct pipe + # for the IFS="#" to work we need to replace options spaces with exactly ONE # + $(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') ${FOLDERS[*]} 2>&1 || echo "[!] Borg backup aborted."; + fi; + # remove the temporary exclude file if it exists + if [ -f "${TMP_EXCLUDE_FILE}" ]; then + rm -f "${TMP_EXCLUDE_FILE}"; + fi; +else + echo "[! $(date +'%F %T')] No folders where set for the backup"; + exit 1; +fi; + +# clean up, always verbose +echo "--- [PRUNE : $(date +'%F %T')] ------------------------------------------->"; +# build command +COMMAND="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} ${KEEP_OPTIONS[*]} ${REPOSITORY}"; +echo "Prune repository with keep${KEEP_INFO:1}"; +if [ ${DEBUG} -eq 1 ]; then + echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g'; +fi; +# for the IFS="#" to work we need to replace options spaces with exactly ONE # +$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted"; + +. "${DIR}/borg.backup.functions.close.sh"; + +# __END__ diff --git a/borg.backup.functions.check.sh b/borg.backup.functions.check.sh new file mode 100644 index 0000000..0f19dba --- /dev/null +++ b/borg.backup.functions.check.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash + +# start time in seconds +START=$(date +'%s'); +# start logging from here +exec &> >(tee -a "${LOG}"); +echo "=== [START : $(date +'%F %T')] ==[${MODULE}]====================================>"; +# show info for version always +echo "Script version: ${VERSION}"; +# show type +echo "Backup module : ${MODULE}"; +echo "Module version: ${MODULE_VERSION}"; +# show base folder always +echo "Base folder : ${BASE_FOLDER}"; + +# if force check is true set CHECK to 1unless INFO is 1 +# Needs bash 4.0 at lesat for this +if [ "${FORCE_CHECK,,}" = "true" ] && [ ${INFO} -eq 0 ]; then + CHECK=1; + if [ ${DEBUG} -eq 1 ]; then + echo "Force repository check"; + fi; +fi; + +# remote borg path +if [ ! -z "${TARGET_BORG_PATH}" ]; then + if [[ "${TARGET_BORG_PATH}" =~ \ |\' ]]; then + echo "Space found in ${TARGET_BORG_PATH}. Aborting"; + echo "There are issues with passing on paths with spaces" + echo "as parameters" + exit; + fi; + OPT_REMOTE="--remote-path="$(printf "%q" "${TARGET_BORG_PATH}"); +fi; + +if [ -z "${TARGET_FOLDER}" ]; then + echo "[! $(date +'%F %T')] No target folder has been set yet"; + exit 1; +else + # There are big issues with TARGET FOLDERS with spaces + # we should abort anything with this + if [[ "${TARGET_FOLDER}" =~ \ |\' ]]; then + echo "Space found in ${TARGET_FOLDER}. Aborting"; + echo "There is some problem with passing paths with spaces as"; + echo "repository base folder" + exit; + fi; + + # This does not care for multiple trailing or leading slashes + # it just makes sure we have at least one set + # for if we have a single slash, remove it + TARGET_FOLDER=${TARGET_FOLDER%/} + TARGET_FOLDER=${TARGET_FOLDER#/} + # and add slash front and back and escape the path + TARGET_FOLDER=$(printf "%q" "/${TARGET_FOLDER}/"); +fi; + +# if we have user/host then we build the ssh command +TARGET_SERVER=''; +# allow host only (if full setup in .ssh/config) +# user@host OR ssh://user@host:port/ IF TARGET_PORT is set +# user/host/port +if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then + TARGET_SERVER="ssh://${TARGET_USER}@${TARGET_HOST}:${TARGET_PORT}/"; +# host/port +elif [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then + TARGET_SERVER="ssh://${TARGET_HOST}:${TARGET_PORT}/"; +# user/host +elif [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then + TARGET_SERVER="${TARGET_USER}@${TARGET_HOST}:"; +# host +elif [ ! -z "${TARGET_HOST}" ]; then + TARGET_SERVER="${TARGET_HOST}:"; +fi; +# we dont allow special characters, so we don't need to special escape it +REPOSITORY="${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}"; +echo "Repository : ${REPOSITORY}"; + +# check compression if given is valid and check compression level is valid if given +if [ ! -z "${COMPRESSION}" ]; then + # valid compression + if [ "${COMPRESSION}" = "lz4" ] || [ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ] || [ "${COMPRESSION}" = "zstd" ]; then + OPT_COMPRESSION="-C=${COMPRESSION}"; + # if COMPRESSION_LEVEL, check it is a valid regex + # for zlib, zstd, lzma + if [ ! -z "${COMPRESSION_LEVEL}" ] && ([ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ] || [ "${COMPRESSION}" = "zstd" ]); then + MIN_COMPRESSION=0; + MAX_COMPRESSION=0; + case "${COMPRESSION}" in + zlib|lzma) + MIN_COMPRESSION=0; + MAX_COMPRESSION=9; + ;; + zstd) + MIN_COMPRESSION=1; + MAX_COMPRESSION=22; + ;; + *) + MIN_COMPRESSION=0; + MAX_COMPRESSION=0; + ;; + esac; + # if [ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ] + # MIN_COMPRESSION=0; + # MAX_COMPRESSION=9; + # elif [ "${COMPRESSION}" = "zstd" ]; then + # MIN_COMPRESSION=1; + # MAX_COMPRESSION=22; + # fi; + error_message="[! $(date +'%F %T')] Compression level for ${COMPRESSION} needs to be a numeric value between ${MIN_COMPRESSION} and ${MAX_COMPRESSION}: ${COMPRESSION_LEVEL}"; + if ! [[ "${COMPRESSION_LEVEL}" =~ ${REGEX_NUMERIC} ]]; then + echo ${error_message}; + exit 1; + elif [ ${COMPRESSION_LEVEL} -lt ${MIN_COMPRESSION} ] || [ ${COMPRESSION_LEVEL} -gt ${MAX_COMPRESSION} ]; then + echo ${error_message}; + exit 1; + else + OPT_COMPRESSION=${OPT_COMPRESSION}","${COMPRESSION_LEVEL}; + fi; + fi; + else + echo "[! $(date +'%F %T')] Compress setting need to be lz4, zstd, zlib or lzma. Or empty for no compression: ${COMPRESSION}"; + exit 1; + fi; +fi; + +# home folder, needs to be set if there is eg a HOME=/ in the crontab +if [ ! -w "${HOME}" ] || [ "${HOME}" = '/' ]; then + HOME=$(eval echo "$(whoami)"); +fi; + +# build options and info string, +# also flag BACKUP_SET check if hourly is set +KEEP_OPTIONS=(); +KEEP_INFO=""; +BACKUP_SET_CHECK=0; +if [ ${KEEP_LAST} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}"); + KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}"; +fi; +if [ ${KEEP_HOURS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}"); + KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}"; + BACKUP_SET_CHECK=1; +fi; +if [ ${KEEP_DAYS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}"); + KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}"; +fi; +if [ ${KEEP_WEEKS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}"); + KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}"; +fi; +if [ ${KEEP_MONTHS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}"); + KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}"; +fi; +if [ ${KEEP_YEARS} -gt 0 ]; then + KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}"); + KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}"; +fi; +if [ ! -z "${KEEP_WITHIN}" ]; then + # check for invalid string. can only be number + H|d|w|m|y + if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then + KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}"); + KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}"; + if [[ "${KEEP_WITHIN}" == *"H"* ]]; then + BACKUP_SET_CHECK=1; + fi; + else + echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string."; + exit 1; + fi; +fi; +# abort if KEEP_OPTIONS is empty +if [ -z "${KEEP_OPTIONS}" ]; then + echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format."; + exit 1; +fi; +# set BACKUP_SET if empty, check for for DATE is set +if [ -z "${BACKUP_SET}" ]; then + # DATE is deprecated and will be removed + if [ ! -z "${DATE}" ]; then + echo "[!] DEPRECATED: The use of DATE variable is deprecated, use BACKUP_SET instead"; + BACKUP_SET="${DATE}"; + else + # default + BACKUP_SET="{now:%Y-%m-%d}"; + fi; +fi; +# backup set check, and there is no hour entry (%H) in the archive string +# we add T%H:%M:%S in this case, before the last } +if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then + BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/"); +fi; + +# for folders list split set to "#" and keep the old setting as is +_IFS=${IFS}; +IFS="#"; +# turn off for non file +if [ "${MODULE}" != "file" ]; then + IFS=${_IFS}; +fi; +# general borg settings +# set base path to config directory to keep cache/config separated +export BORG_BASE_DIR="${BASE_FOLDER}"; +# ignore non encrypted access +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=${_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK}; +# ignore moved repo access +export BORG_RELOCATED_REPO_ACCESS_IS_OK=${_BORG_RELOCATED_REPO_ACCESS_IS_OK}; +# and for debug print that tout +if [ ${DEBUG} -eq 1 ]; then + echo "export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=${_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK};"; + echo "export BORG_RELOCATED_REPO_ACCESS_IS_OK=${_BORG_RELOCATED_REPO_ACCESS_IS_OK};"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; +fi; +# prepare debug commands only +COMMAND_EXPORT="export BORG_BASE_DIR=\"${BASE_FOLDER}\";" +COMMAND_INFO="${COMMAND_EXPORT}borg info ${OPT_REMOTE} ${REPOSITORY}"; +# if the is not there, call init to create it +# if this is user@host, we need to use ssh command to check if the file is there +# else a normal check is ok +# unless explicit given, check is skipped +if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then + echo "--- [CHECK : $(date +'%F %T')] ------------------------------------------->"; + if [ ! -z "${TARGET_SERVER}" ]; then + if [ ${DEBUG} -eq 1 ]; then + echo "borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\""; + fi; + # use borg info and check if it returns "Repository ID:" in the first line + REPO_CHECK=$(borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:"); + # this is currently a hack to work round the error code in borg info + # this checks if REPO_CHECK holds this error message and then starts init + regex="^Some part of the script failed with an error:"; + if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${regex} ]]; then + INIT_REPOSITORY=1; + fi; + elif [ ! -d "${REPOSITORY}" ]; then + INIT_REPOSITORY=1; + fi; + # if check but no init and repo is there but init file is missing set it + if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 0 ] && + [ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then + # write init file + echo "[!] Add missing init check file"; + echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}"; + fi; + # end if checked but repository is not here + if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then + echo "[! $(date +'%F %T')] No repository. Please run with -I flag to initialze repository"; + exit 1; + fi; + if [ ${EXIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ]; then + echo "Repository exists"; + echo "For more information run:" + echo "${COMMAND_INFO}"; + echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + exit; + fi; +fi; +if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then + echo "--- [INIT : $(date +'%F %T')] ------------------------------------------->"; + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}"; + fi + if [ ${DRYRUN} -eq 0 ]; then + # should trap and exit properly here + borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}; + # write init file + echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}"; + echo "Repository initialized"; + echo "For more information run:" + echo "${COMMAND_INFO}"; + fi + echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + # exit after init + exit; +elif [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 0 ]; then + echo "[! $(date +'%F %T')] Repository already initialized"; + echo "For more information run:" + echo "${COMMAND_INFO}"; + exit 1; +fi; + +# check for init file +if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then + echo "[! $(date +'%F %T')] It seems the repository has never been initialized." + echo "Please run -I to initialize or if already initialzed run with -C for init update." + exit 1; +fi; + +# PRINT OUT current data, only do this if REPO exists +if [ ${PRINT} -eq 1 ]; then + echo "--- [PRINT : $(date +'%F %T')] ------------------------------------------->"; + FORMAT="{archive} {comment:6} {start} - {end} [{id}] ({username}@{hostname}){NL}" + # show command on debug or dry run + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}"; + fi; + # run info command if not a dry drun + if [ ${DRYRUN} -eq 0 ]; then + borg list ${OPT_REMOTE} --format "${FORMAT}" ${REPOSITORY} ; + fi; + if [ ${VERBOSE} -eq 1 ]; then + echo ""; + echo "Base command info:" + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]"; + echo "Replace [COMMAND] with list for listing or extract for restoring backup data." + echo "Replace [BACKUP] with archive name." + echo "If no [PATH] is given then all files will be restored." + echo "Before extracting -n (dry run) is recommended to use." + echo "If archive size is needed the info command with archive name has to be used." + echo "When listing (list) data the --format command can be used." + echo "Example: \"{mode} {user:6} {group:6} {size:8d} {csize:8d} {dsize:8d} {dcsize:8d} {mtime} {path}{extra} [{health}]{NL}\"" + else + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]"; + fi; + exit; +fi; + +# __END__ diff --git a/borg.backup.functions.close.sh b/borg.backup.functions.close.sh new file mode 100644 index 0000000..07e1d79 --- /dev/null +++ b/borg.backup.functions.close.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# unset borg settings +unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK +DURATION=$[ $(date +'%s')-$START ]; +echo "=== [Run time: $(convert_time ${DURATION})]"; +echo "=== [END : $(date +'%F %T')] ==[${MODULE}]===================>"; + +# __END__ diff --git a/borg.backup.functions.info.sh b/borg.backup.functions.info.sh new file mode 100644 index 0000000..33e612a --- /dev/null +++ b/borg.backup.functions.info.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +if [ ${INFO} -eq 1 ]; then + echo "--- [INFO : $(date +'%F %T')] ------------------------------------------->"; + # show command on debug or dry run + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg info ${OPT_REMOTE} ${REPOSITORY}"; + fi; + # run info command if not a dry drun + if [ ${DRYRUN} -eq 0 ]; then + borg info ${OPT_REMOTE} ${REPOSITORY}; + fi; + if [ "${MODULE}" = "files" ]; then + if [ $FOLDER_OK -eq 1 ]; then + echo "--- [Run command]:"; + #IFS="#"; + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${COMMAND} "${FOLDERS_Q[*]}; + else + echo "[!] No folders where set for the backup"; + fi; + # remove the temporary exclude file if it exists + if [ -f "${TMP_EXCLUDE_FILE}" ]; then + rm -f "${TMP_EXCLUDE_FILE}"; + fi; + fi; + echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>"; + exit; +fi; + +# __END__ diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh new file mode 100644 index 0000000..d76564f --- /dev/null +++ b/borg.backup.functions.init.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash + +set -ETu #-e -o pipefail +trap cleanup SIGINT SIGTERM ERR + +cleanup() { + # script cleanup here + echo "Some part of the script failed with an error: $? @LINE: $(caller)"; + # unset exported vars + unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK; + # end trap + trap - SIGINT SIGTERM ERR +} +# on exit unset any exported var +trap "unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK" EXIT; + +# version for all general files +VERSION="3.0.0"; + +# default log folder if none are set in config or option +_LOG_FOLDER="/var/log/borg.backup/"; +# log file name is set based on BACKUP_FILE, .log is added +LOG_FOLDER=""; +# should be there on everything +TEMPDIR="/tmp/"; +# creates borg backup based on the include/exclude files +# if base borg folder (backup files) does not exist, it will automatically init it +# base folder +BASE_FOLDER="/usr/local/scripts/borg/"; +# base settings and init flag +SETTINGS_FILE="borg.backup.settings"; +# include files +INCLUDE_FILE=""; +EXCLUDE_FILE=""; +# backup folder initialzed check +BACKUP_INIT_CHECK=""; +# debug/verbose +VERBOSE=0; +LIST=0; +DEBUG=0; +DRYRUN=0; +INFO=0; +CHECK=0; +INIT=0; +EXIT=0; +PRINT=0; +# flags, set to no to disable +_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK="yes"; +_BORG_RELOCATED_REPO_ACCESS_IS_OK="yes"; +# other variables +TARGET_SERVER=""; +REGEX=""; +REGEX_COMMENT="^[\ \t]*#"; +REGEX_GLOB='\*'; +REGEX_NUMERIC="^[0-9]{1,2}$"; +PRUNE_DEBUG=""; +INIT_REPOSITORY=0; +FOLDER_OK=0; +TMP_EXCLUDE_FILE=""; +# opt flags +OPT_VERBOSE=""; +OPT_PROGRESS=""; +OPT_LIST=""; +OPT_REMOTE=""; +OPT_LOG_FOLDER=""; +OPT_EXCLUDE=""; +# config variables (will be overwritten from .settings file) +TARGET_USER=""; +TARGET_HOST=""; +TARGET_PORT=""; +TARGET_BORG_PATH=""; +TARGET_FOLDER=""; +BACKUP_FILE=""; +SUB_BACKUP_FILE=""; +# lz4, zstd 1-22 (3), zlib 0-9 (6), lzma 0-9 (6) +COMPRESSION="zstd"; +COMPRESSION_LEVEL=3; +SUB_COMPRESSION=""; +SUB_COMPRESSION_LEVEL=""; +# encryption settings +ENCRYPTION="none"; +# force check always +FORCE_CHECK="false"; +DATE=""; # to be deprecated +BACKUP_SET=""; +SUB_BACKUP_SET=""; +# for database backup only +DATABASE_FULL_DUMP=""; +DATABASE_USER=""; +# only for mysql old config file +MYSQL_DB_CONFIG=""; +MYSQL_DB_CONFIG_PARAM=""; +# default keep 7 days, 4 weeks, 6 months +# if set 0, ignore +# note that for last/hourly it is needed to create a different +# BACKUP SET that includes hour and minute information +# IF BACKUP_SET is empty, this is automatically added +# general keep last, if only this is set only last n will be kept +KEEP_LAST=0; +KEEP_HOURS=0; +KEEP_DAYS=7; +KEEP_WEEKS=4; +KEEP_MONTHS=6; +KEEP_YEARS=1; +# in the format of nY|M|d|h|m|s +KEEP_WITHIN=""; +# sub override init to empty +SUB_KEEP_LAST=""; +SUB_KEEP_HOURS=""; +SUB_KEEP_DAYS=""; +SUB_KEEP_WEEKS=""; +SUB_KEEP_MONTHS=""; +SUB_KEEP_YEARS=""; +SUB_KEEP_WITHIN=""; + +function usage() +{ + cat <<- EOT + Usage: ${0##/*/} [-c ] [-v] [-d] + + -c : if this is not given, ${BASE_FOLDER} is used + -L : override config set and default log folder + -P: print list of archives created + -C: check if repository exists, if not abort + -E: exit after check + -I: init repository (must be run first) + -v: be verbose + -i: print out only info + -l: list files during backup + -d: debug output all commands + -n: only do dry run + -h: this help page + + Version : ${VERSION} + Module Version: ${MODULE_VERSION} + Module : ${MODULE} + EOT +} + +# set options +while getopts ":c:L:vldniCEIPh" opt; do + case "${opt}" in + c|config) + BASE_FOLDER=${OPTARG}; + ;; + L|log) + OPT_LOG_FOLDER=${OPTARG}; + ;; + C|Check) + # will check if repo is there and abort if not + CHECK=1; + ;; + E|Exit) + # exit after check + EXIT=1; + ;; + I|Init) + # will check if there is a repo and init it + # previoous this was default + CHECK=1; + INIT=1; + ;; + P|Print) + # use borg list to print list of archves + PRINT=1; + ;; + v|verbose) + VERBOSE=1; + ;; + l|list) + LIST=1; + ;; + i|info) + INFO=1; + ;; + d|debug) + DEBUG=1; + ;; + n|dryrun) + DRYRUN=1; + ;; + h|help) + usage; + exit; + ;; + :) + echo "Option -$OPTARG requires an argument." + ;; + \?) + echo -e "\n Option does not exist: ${OPTARG}\n"; + usage; + exit 1; + ;; + esac; +done; + +# add trailing slasd for base folder +[[ "${BASE_FOLDER}" != */ ]] && BASE_FOLDER=${BASE_FOLDER}"/"; +# must have settings file there, if not, abort early +if [ ! -f "${BASE_FOLDER}${SETTINGS_FILE}" ]; then + echo "No settings file could be found: ${BASE_FOLDER}${SETTINGS_FILE}"; + exit 1; +fi; +if [ ! -w "${BASE_FOLDER}" ]; then + echo "Cannot write to BASE_FOLDER ${BASE_FOLDER}"; + echo "Is the group set to 'backup' and is this group allowed to write?" + echo "chgrp -R backup ${BASE_FOLDER}"; + echo "chmod -R g+rws ${BASE_FOLDER}"; + exit 1; +fi; + +# info -i && -C/-I cannot be run together +if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${INFO} -eq 1 ]; then + echo "Cannot have -i info option and -C check or -I initialize option at the same time"; + exit 1; +fi; +# print -P cannot be run with -i/-C/-I together +if [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INFO} -eq 1 ]; then + echo "Cannot have -P print option and -i info, -C check or -I initizalize option at the same time"; + exit 1; +fi; + +# verbose & progress +if [ ${VERBOSE} -eq 1 ]; then + OPT_VERBOSE="-v"; + OPT_PROGRESS="-p"; +fi; +# list files +if [ ${LIST} -eq 1 ]; then + OPT_LIST="--list"; +fi; +if [ ${DRYRUN} -eq 1 ]; then + PRUNE_DEBUG="--dry-run"; +fi; + +# read config file +. "${BASE_FOLDER}${SETTINGS_FILE}"; +# ** SUB LOAD +# a settings file always end in .settings, replace that with lower case MODULE.settings +SETTINGS_FILE_SUB=$(echo "${SETTINGS_FILE}" | sed -e "s/\.settings/\.${MODULE,,}\.settings/"); +# if mysql/pgsql run, load sub settings +if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then + . "${BASE_FOLDER}${SETTINGS_FILE_SUB}"; + # if SUB_ set override master + if [ ! -z "${SUB_BACKUP_FILE}" ]; then + BACKUP_FILE=${SUB_BACKUP_FILE} + fi; + # add module name to backup file, always + BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg}; + # if sub backup set it set, override current + if [ ! -z "${SUB_BACKUP_SET}" ]; then + BACKUP_SET=${SUB_BACKUP_SET}; + fi; + # ovrride compression + if [ ! -z "${SUB_COMPRESSION}" ]; then + COMPRESSION=${SUB_COMPRESSION}; + fi; + if [ ! -z "${SUB_COMPRESSION_LEVEL}" ]; then + COMPRESSION_LEVEL=${SUB_COMPRESSION_LEVEL}; + fi; + # check override for keep time + if [ ! -z "${SUB_KEEP_LAST}" ]; then + KEEP_LAST=${SUB_KEEP_LAST}; + fi; + if [ ! -z "${SUB_KEEP_HOURS}" ]; then + KEEP_HOURS=${SUB_KEEP_HOURS}; + fi; + if [ ! -z "${SUB_KEEP_DAYS}" ]; then + KEEP_DAYS=${SUB_KEEP_DAYS}; + fi; + if [ ! -z "${SUB_KEEP_WEEKS}" ]; then + KEEP_WEEKS=${SUB_KEEP_WEEKS}; + fi; + if [ ! -z "${SUB_KEEP_YEARS}" ]; then + KEEP_YEARS=${SUB_KEEP_YEARS}; + fi; + if [ ! -z "${SUB_KEEP_LAST}" ]; then + KEEP_LAST=${SUB_KEEP_LAST}; + fi; + if [ ! -z "${SUB_KEEP_WITHIN}" ]; then + KEEP_WITHIN=${SUB_KEEP_WITHIN}; + fi; +fi; +# backup file must be set +if [ -z "${BACKUP_FILE}" ]; then + echo "No BACKUP_FILE set"; + exit; +fi; +# backup file (folder) must end as .borg +# BACKUP FILE also cannot start with / or have / inside or start with ~ +# valid file name check, alphanumeric, -,._ ... +if ! [[ "${BACKUP_FILE}" =~ ^[A-Za-z0-9,._-]+\.borg$ ]]; then + echo "BACKUP_FILE ${BACKUP_FILE} can only contain A-Z a-z 0-9 , . _ - chracters and must end with .borg"; + exit 1; +fi; +# error if the repository file still has the default name +# This is just for old sets +REGEX="^some\-prefix\-"; +if [[ "${BACKUP_FILE}" =~ ${REGEX} ]]; then + echo "[DEPRECATED] The repository name still has the default prefix: ${BACKUP_FILE}"; + exit 1; +fi; + +# check LOG_FOLDER, TARGET_BORG_PATH, TARGET_FOLDER must have no ~/ as start position +if [[ ${LOG_FOLDER} =~ ^~\/ ]]; then + echo "LOG_FOLDER path cannot start with ~/. Path must be absolute: ${LOG_FOLDER}"; + exit 1; +fi; +if [[ ${TARGET_BORG_PATH} =~ ^~\/ ]]; then + echo "TARGET_BORG_PATH path cannot start with ~/. Path must be absolute: ${TARGET_BORG_PATH}"; + exit 1; +fi; +if [[ ${TARGET_FOLDER} =~ ^~\/ ]]; then + echo "TARGET_FOLDER path cannot start with ~/. Path must be absolute: ${TARGET_FOLDER}"; + exit 1; +fi + +# log file set and check +# option folder overrides all other folders +if [ ! -z "${OPT_LOG_FOLDER}" ]; then + LOG_FOLDER="${OPT_LOG_FOLDER}"; +fi; +# if empty folder set to default folder +if [ -z "${LOG_FOLDER}" ]; then + LOG_FOLDER="${_LOG_FOLDER}"; +fi; +# if folder does not exists create it +if [ ! -d "${LOG_FOLDER}" ]; then + mkdir "${LOG_FOLDER}"; +fi; +# set the output log folder +# LOG=$(printf "%q" "${LOG_FOLDER}/${BACKUP_FILE}.log"); +LOG="${LOG_FOLDER}/${BACKUP_FILE}.log"; +# fail if not writeable to folder or file +if [[ -f "${LOG}" && ! -w "${LOG}" ]] || [[ ! -f "${LOG}" && ! -w "${LOG_FOLDER}" ]]; then + echo "Log folder or log file is not writeable: ${LOG}"; + echo "Is the group set to 'backup' and is this group allowed to write?" + echo "chgrp -R backup ${LOG}"; + echo "chmod -R g+rws ${LOG}"; + exit 1; +fi; + +# if ENCRYPTION is empty or not in the valid list fall back to none +if [ -z "${ENCRYPTION}" ]; then + ENCRYPTION="none"; +#else + # TODO check for invalid encryption string +fi; + +## FUNCTIONS + +# METHOD: convert_time +# PARAMS: timestamp in seconds or with milliseconds (nnnn.nnnn) +# RETURN: formated string with human readable time (d/h/m/s) +# CALL : var=$(convert_time $timestamp); +# DESC : converts a timestamp or a timestamp with float milliseconds +# to a human readable format +# output is in days/hours/minutes/seconds +function convert_time +{ + timestamp=${1}; + # round to four digits for ms + timestamp=$(printf "%1.4f" $timestamp); + # get the ms part and remove any leading 0 + ms=$(echo ${timestamp} | cut -d "." -f 2 | sed -e 's/^0*//'); + timestamp=$(echo ${timestamp} | cut -d "." -f 1); + timegroups=(86400 3600 60 1); # day, hour, min, sec + timenames=("d" "h" "m" "s"); # day, hour, min, sec + output=( ); + time_string=; + for timeslice in ${timegroups[@]}; do + # floor for the division, push to output + output[${#output[*]}]=$(awk "BEGIN {printf \"%d\", ${timestamp}/${timeslice}}"); + timestamp=$(awk "BEGIN {printf \"%d\", ${timestamp}%${timeslice}}"); + done; + + for ((i=0; i<${#output[@]}; i++)); do + if [ ${output[$i]} -gt 0 ] || [ ! -z "$time_string" ]; then + if [ ! -z "${time_string}" ]; then + time_string=${time_string}" "; + fi; + time_string=${time_string}${output[$i]}${timenames[$i]}; + fi; + done; + if [ ! -z ${ms} ] && [ ${ms} -gt 0 ]; then + time_string=${time_string}" "${ms}"ms"; + fi; + # just in case the time is 0 + if [ -z "${time_string}" ]; then + time_string="0s"; + fi; + echo -n "${time_string}"; +} + +# __END__ diff --git a/borg.backup.mysql.settings-default b/borg.backup.mysql.settings-default new file mode 100644 index 0000000..408ee92 --- /dev/null +++ b/borg.backup.mysql.settings-default @@ -0,0 +1,11 @@ +# Borg backup wrapper scripts settings + +# override settings in borg.backup.settings with SUB_ prefix +# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_* + +# set to 1 to dump all into one file instead of per database +# note that with this databases that have been dropped need to be pruned manually +# if 'schema' word is used, only schema data is dumped +DATABASE_FULL_DUMP=""; +# db config (for older mysql setups without ident basedlogin) +MYSQL_DB_CONFIG=""; diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh new file mode 100755 index 0000000..32925ca --- /dev/null +++ b/borg.backup.mysql.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# Backup MySQL/MariaDB +# default is per table dump, can be set to one full dump +# config override set in borg.backup.mysql.settings +# if run as mysql user, be sure user is in the backup group + +# Run -I first to initialize repository +# There are no automatic repository checks unless -C is given + +# set last edit date + time +MODULE="mysql" +MODULE_VERSION="1.0.0"; + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +# init system +. "${DIR}/borg.backup.functions.init.sh"; + +# include and exclude file +INCLUDE_FILE="borg.backup.mysql.include"; +EXCLUDE_FILE="borg.backup.mysql.exclude"; +SCHEMA_ONLY_FILE="borg.backup.mysql.schema-only"; +BACKUP_INIT_CHECK="borg.backup.mysql.init"; + +# check valid data +. "${DIR}/borg.backup.functions.check.sh"; +# if info print info and then abort run +. "${DIR}/borg.backup.functions.info.sh"; + +# if there is an DB extra config +# on current installs there should be a root or mysql user with unix socket connection +# the script should by run as the mysql or root user (sudo -u mysql ...) +if [ -f "${MYSQL_DB_CONFIG}" ]; then + # MYSQL_DB_CONFIG='/root/.my.cnf'; + MYSQL_DB_CONFIG_PARAM="--defaults-extra-file=${MYSQL_DB_CONFIG}"; +fi; +MYSQL_BASE_PATH='/usr/bin/'; +MYSQL_DUMP=${MYSQL_BASE_PATH}'mysqldump'; +MYSQL_CMD=${MYSQL_BASE_PATH}'mysql'; +# no dump or mysql, bail +if [ ! -f "${MYSQL_DUMP}" ]; then + echo "[! $(date +'%F %T')] mysqldump binary not found"; + exit 1; +fi; +if [ ! -f "${MYSQL_CMD}" ]; then + echo "[! $(date +'%F %T')] mysql binary not found"; + exit 1; +fi; +# check that the user can actually do, else abort here +# note: this is the only way to not error +_MYSQL_CHECK=$(mysqladmin ${MYSQL_DB_CONFIG_PARAM} ping 2>&1); +_MYSQL_OK=$(echo "${_MYSQL_CHECK}" | grep "is alive"); +if [ -z "${_MYSQL_OK}" ]; then + echo "[! $(date +'%F %T')] Current user has no access right to mysql database"; + exit 1; +fi; +# below is for file name only +# set DB_VERSION (Distrib n.n.n-type) +# NEW: mysql Ver 15.1 Distrib 10.5.12-MariaDB, for debian-linux-gnu (x86_64) using EditLine wrapper +# OLD: mysql Ver 14.14 Distrib 5.7.35, for Linux (x86_64) using EditLine wrapper +_DB_VERSION_TYPE=$("${MYSQL_CMD}" --version); +_DB_VERSION=$(echo "${_DB_VERSION_TYPE}" | sed 's/.*Distrib \([0-9]\{1,\}\.[0-9]\{1,\}\)\.[0-9]\{1,\}.*/\1/'); +DB_VERSION=$(echo "${_DB_VERSION}" | cut -d " " -f 1); +# temporary until correct type detection is set +DB_TYPE="mysql"; +# try to get type from -string, if empty set mysql +# if [[ ${_DB_VERSION_TYPE##*-*} ]]; then +# DB_TYPE="mysql"; +# else +# DB_TYPE=$(echo "${_DB_TYPE}" | sed -e 's/.*[0-9]-\([A-Za-z]\{1,\}\).*/\1/'); +# fi; +DB_PORT='3306'; +DB_HOST='local'; + +# those dbs have to be dropped with skip locks (single transaction) +NOLOCKDB="information_schema performance_schema" +NOLOCKS="--single-transaction" +# those tables need to be dropped with EVENTS +EVENTDB="mysql" +EVENTS="--events" + +# borg call, replace ##...## parts +_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; +_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; + +# ALL IN ONE FILE or PER DATABASE FLAG +if [ ! -z "${DATABASE_FULL_DUMP}" ]; then + SCHEMA_ONLY=''; + schema_flag='data'; + if [ "${DATABASE_FULL_DUMP}" = "schema" ]; then + SCHEMA_ONLY='--no-data'; + schema_flag='schema'; + fi; + echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # We only do a full backup and not per table backup here + # Filename + FILENAME="all-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql" + # backup set: + BACKUP_SET_NAME="all-${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="all-"; + # borg call + BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; + echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}"; + echo "${BORG_PRUNE}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + ${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}; + _backup_error=$?; + if [ $_backup_error -ne 0 ]; then + echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}"; + exit $_backup_error; + fi; + fi; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; +else + ${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" | + while read db; do + echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # exclude checks + include=0; + if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then + while read incl_db; do + if [ "${db}" = "${incl_db}" ]; then + include=1; + break; + fi; + done<"${BASE_FOLDER}${INCLUDE_FILE}"; + else + include=1; + fi; + exclude=0; + if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then + while read excl_db; do + if [ "${db}" = "${excl_db}" ]; then + exclude=1; + break; + fi; + done<"${BASE_FOLDER}${EXCLUDE_FILE}"; + fi; + if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then + # lock check + nolock=''; + for nolock_db in $NOLOCKDB; + do + if [ "$nolock_db" = "$db" ]; + then + nolock=$NOLOCKS; + fi; + done; + # event check + event=''; + for event_db in $EVENTDB; + do + if [ "$event_db" = "$db" ]; + then + event=$EVENTS; + fi; + done; + # set dump type + SCHEMA_ONLY=''; # empty for all + schema_flag='data'; # or data + if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then + while read schema_db; do + if [ "${db}" = "${schema_db}" ]; then + SCHEMA_ONLY='--no-data'; + schema_flag='schema'; + # skip out + break; + fi; + done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}"; + fi; + # prepare borg calls + FILENAME="${db}-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql" + # backup set: + BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${db}-" + # borg call + BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + # debug or dry run + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; + echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} $nolock $event --opt ${SCHEMA_ONLY} --add-drop-database --databases ${db} | ${BORG_CALL};" + fi; + # backup + if [ ${DRYRUN} -eq 0 ]; then + $MYSQL_DUMP ${MYSQL_DB_CONFIG_PARAM} $nolock $event --opt ${SCHEMA_ONLY} --add-drop-database --databases ${db} | ${BORG_CALL}; + _backup_error=$?; + if [ $_backup_error -ne 0 ]; then + echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + exit $_backup_error; + fi; + fi; + echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + else + echo "- [E] ${db}"; + fi; + done; +fi; + +. "${DIR}/borg.backup.functions.close.sh"; + +# __END__ diff --git a/borg.backup.pgsql.settings-default b/borg.backup.pgsql.settings-default new file mode 100644 index 0000000..2019f1d --- /dev/null +++ b/borg.backup.pgsql.settings-default @@ -0,0 +1,11 @@ +# Borg backup wrapper scripts settings + +# override settings in borg.backup.settings with SUB_ prefix +# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_* + +# set to 1 to dump all into one file instead of per database +# note that with this databases that have been dropped need to be pruned manually +# if 'schema' word is used, only schema data is dumped +DATABASE_FULL_DUMP=""; +# override default postgres user +DATABASE_USER=""; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh new file mode 100755 index 0000000..1a9dedf --- /dev/null +++ b/borg.backup.pgsql.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash + +# Backup PostgreSQL +# default is per table dump, can be set to one full dump +# config override set in borg.backup.pgsql.settings +# if run as postgres user, be sure user is in the backup group + +# Run -I first to initialize repository +# There are no automatic repository checks unless -C is given + +# set last edit date + time +MODULE="pgsql" +MODULE_VERSION="0.1.0"; + + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +# init system +. "${DIR}/borg.backup.functions.init.sh"; + +# include and exclude file +INCLUDE_FILE="borg.backup.pgsql.include"; +EXCLUDE_FILE="borg.backup.pgsql.exclude"; +SCHEMA_ONLY_FILE="borg.backup.pgsql.schema-only"; +BACKUP_INIT_CHECK="borg.backup.pgsql.init"; + +# check valid data +. "${DIR}/borg.backup.functions.check.sh"; +# if info print info and then abort run +. "${DIR}/borg.backup.functions.info.sh"; + +if [ ! -z "${DATABASE_USER}" ]; then + DB_USER=${DATABASE_USER}; +else + DB_USER='postgres'; +fi; +# get current pgsql version first +# if first part <10 then user full, else only first part +# eg 9.4 -> 9.4, 12.5 -> 12 +PG_VERSION=$(pgv=$(psql -U ${DB_USER} -d template1 -t -A -F "," -X -q -c 'select version();' | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/'); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi ); +_PATH_PG_VERSION=${PG_VERSION}; +_backup_error=$?; +if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then + echo "[! $(date +'%F %T')] Cannot get PostgreSQL server version: ${_backup_error}"; + exit $_backup_error; +fi; + +# path set per Distribution type and current running DB version +# Redhat: PG_BASE_PATH='/usr/pgsql-'; +# AWS 1: PG_BASE_PATH='/usr/lib64/pgsql'; +# Debian: PG_BASE_PATH='/usr/lib/postgresql/'; +PG_BASE_PATH='/usr/lib/postgresql/'; +if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then + PG_BASE_PATH='/usr/pgsql-'; + if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then + PG_BASE_PATH='/usr/lib64/pgsql'; + _PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//'): + if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then + echo "[! $(date +'%F %T')] PostgreSQL not found in any paths"; + exit 1; + fi; + fi; +fi; +PG_PATH=${PG_BASE_PATH}${_PATH_PG_VERSION}'/bin/'; +PG_PSQL=${PG_PATH}'psql'; +PG_DUMP=${PG_PATH}'pg_dump'; +PG_DUMPALL=${PG_PATH}'pg_dumpall'; +# check that command are here +if [ ! -f "${PG_PSQL}" ]; then + echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}"; + exit 1; +fi; +if [ ! -f "${PG_DUMP}" ]; then + echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}"; + exit 1; +fi; +if [ ! -f "${PG_DUMPALL}" ]; then + echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}"; + exit 1; +fi; + +DB_VERSION=${PG_VERSION}; +# TODO override port/host info +DB_PORT='5432'; +DB_HOST='local'; # or +CONN_DB_HOST=''; # -h +CONN_DB_PORT=''; # -p + +# borg call, replace ##...## parts +_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; +_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; + +# ALL IN ONE FILE or PER DATABASE FLAG +if [ ! -z "${DATABASE_FULL_DUMP}" ]; then + SCHEMA_ONLY=''; + schema_flag='data'; + if [ "${DATABASE_FULL_DUMP}" = "schema" ]; then + SCHEMA_ONLY='-s'; + schema_flag='schema'; + fi; + echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # Filename + FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" + # backup set: + BACKUP_SET_NAME="all-${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="all-"; + # borg call + BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; + echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}"; + echo "${BORG_PRUNE}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + $(${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}); + _backup_error=$?; + if [ $_backup_error -ne 0 ]; then + echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}"; + exit $_backup_error; + fi; + fi; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; +else + # dump globals first + db="pg_globals"; + schema_flag="data"; + echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + # Filename + FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" + # backup set: + BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}"; + BACKUP_SET_PREFIX="${db}-"; + # borg call + BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; + echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}"; + echo "${BORG_PRUNE}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + ${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}; + _backup_error=$?; + if [ $_backup_error -ne 0 ]; then + echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + exit $_backup_error; + fi; + fi; + echo "Prune repository with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + + # get list of tables + for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do + # get the user who owns the DB too + owner=$(echo ${owner_db} | cut -d "," -f 1); + db=$(echo ${owner_db} | cut -d "," -f 2); + encoding=$(echo ${owner_db} | cut -d "," -f 3); + echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; + include=0; + if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then + while read incl_db; do + if [ "${db}" = "${incl_db}" ]; then + include=1; + break; + fi; + done<"${BASE_FOLDER}${INCLUDE_FILE}"; + else + include=1; + fi; + exclude=0; + if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then + while read excl_db; do + if [ "${db}" = "${excl_db}" ]; then + exclude=1; + break; + fi; + done<"${BASE_FOLDER}${EXCLUDE_FILE}"; + fi; + if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then + # set dump type + SCHEMA_ONLY=''; # empty for all + schema_flag='data'; # or data + if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then + while read schema_db; do + if [ "${db}" = "${schema_db}" ]; then + SCHEMA_ONLY='-s'; + schema_flag='schema'; + # skip out + break; + fi; + done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}"; + fi; + # Filename + # Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql + FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" + # backup set: + BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}"; + # PER db either data or schema + BACKUP_SET_PREFIX="${db}-"; + # borg call + BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + # borg prune + BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; + echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}"; + echo "${BORG_PRUNE}"; + fi; + if [ ${DRYRUN} -eq 0 ]; then + ${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}; + _backup_error=$?; + if [ $_backup_error -ne 0 ]; then + echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; + exit $_backup_error; + fi; + fi; + echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}"; + ${BORG_PRUNE}; + else + echo "- [E] ${db}"; + fi; + done; +fi; + +. "${DIR}/borg.backup.functions.close.sh"; + +# __END__ diff --git a/borg.backup.settings-default b/borg.backup.settings-default new file mode 100644 index 0000000..aa3924c --- /dev/null +++ b/borg.backup.settings-default @@ -0,0 +1,40 @@ +# Borg backup wrapper scripts settings + +# NOTE: ALL FOLDERS MUST BE ABSOLUTE WITH NO ~/ AT THE BEGINNING + +# set log folder, if empty default will be used (/var/log/borg.backup) +LOG_FOLDER=""; +# SSH user, host, port +# if ssh config is set only host needs to be filled +TARGET_USER=""; +TARGET_HOST=""; +TARGET_PORT=""; +# if borg is not in default path, for ssh backup only +TARGET_BORG_PATH=""; +# folder where the backup folder will be created +TARGET_FOLDER=""; +# the backup file (folder) for this host $(hostname), must end with .borg +BACKUP_FILE=""; +# compression settings (empty for none, lz4, zstd, zlib, lzma) +# level, if empty then default, else number between 0 and 9, or 1 to 22 for zstd +# default is zstd, 3 +COMPRESSION=""; +COMPRESSION_LEVEL=""; +# encryption settings: +# SHA-256: 'none', 'authenticated', 'repokey', 'keyfile' +# BLAKE2b: 'authenticated-blake2', 'repokey-blake2', 'keyfile-blake2' +# Note: none or empty does not encrypt +# Blank passwords allowed for only key (if used, use keyfile) +# See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically +ENCRYPTION=""; +# force repository check, default is off, set to true for check +FORCE_CHECK=""; +# default is %Y-%m-%d +# todays date, if more than one per day add -%H%M for hour/minute +# it can also be "{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f}" +BACKUP_SET=""; +# prune times, how many are kept in each time frame +KEEP_DAYS=7; +KEEP_WEEKS=4; +KEEP_MONTHS=6; +KEEP_YEARS=1; diff --git a/borg.mount.sh b/borg.mount.sh new file mode 100755 index 0000000..f1f7962 --- /dev/null +++ b/borg.mount.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -e -u -o pipefail + +# mount this servers borg backup to a folder +# if no command is given the target folder is /mnt/restore +# if this folder does not exist, script will exit with an error + +# base folder +BASE_FOLDER="/usr/local/scripts/backup/"; +# borg settings file +SETTINGS_FILE="borg.backup.settings"; +# base mount path (default) +MOUNT_PATH="/mnt/restore/"; +# backup path to borg storage +ATTIC_BACKUP_FILE=''; +# if we are mount or unmount (default is mount) +UMOUNT=0; + +function usage () +{ + cat <<- EOT + Usage: ${0##/*/} [-c ] [-m ] [-f ] [-u : if this is not given, ${BASE_FOLDER} is used + -m : where to mount the image + -u umount mounted image + -f : override full path to backup file instead of using the settings info + EOT +} + +# set options +while getopts ":c:m:uf:h" opt do + case "${opt}" in + c|config) + BASE_FOLDER=${OPTARG}; + ;; + m|mount) + MOUNT_PATH=${OPTARG}; + ;; + u|umount) + UMOUNT=1; + ;; + f|file) + ATTIC_BACKUP_FILE=${OPTARG}; + ;; + h|help) + usage; + exit 0; + ;; + :) + echo "Option -$OPTARG requires an argument." + ;; + \?) + echo -e "\n Option does not exist: ${OPTARG}\n"; + usage; + exit 1; + ;; + esac; +done; + +if [ ! -d "${MOUNT_PATH}" ]; then + echo "The mount path ${MOUNT_PATH} cannot be found"; + exit 0; +fi; + +# add trailing slahd for base folder +[[ "${BASE_FOLDER}" != */ ]] && BASE_FOLDER="${BASE_FOLDER}/"; + +if [ ${UMOUNT} -eq 0 ]; then + TARGET_SERVER=''; + if [ -z "${ATTIC_BACKUP_FILE}" ]; then + if [ ! -f "${BASE_FOLDER}${SETTINGS_FILE}" ]; then + echo "Cannot find ${BASE_FOLDER}${SETTINGS_FILE}"; + exit 0; + fi; + . ${BASE_FOLDER}${SETTINGS_FILE} + # set the borg backup file base on the settings data + # if we have user/host then we build the ssh command + if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then + TARGET_SERVER=${TARGET_USER}"@"${TARGET_HOST}":"; + fi; + REPOSITORY=${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}; + else + REPOSITORY=${ATTIC_BACKUP_FILE}; + fi; + + # check that the repostiory exists + REPOSITORY_OK=0; + if [ ! -z "${TARGET_SERVER}" ]; then + # remove trailing : for this + TARGET_SERVER=${TARGET_SERVER/:}; + # use ssh command to check remote existense + if [ `ssh "${TARGET_SERVER}" "if [ -d \"${TARGET_FOLDER}${BACKUP_FILE}\" ]; then echo 1; else echo 0; fi;"` -eq 1 ]; then + REPOSITORY_OK=1; + fi; + elif [ -d "${REPOSITORY}" ]; then + REPOSITORY_OK=1; + fi; + + if [ ${REPOSITORY_OK} -eq 0 ]; then + echo "Repository ${REPOSITORY} does not exists"; + exit 0; + fi; + + echo "Mounting ${REPOSITORY} on ${MOUNT_PATH}"; + # all ok, lets mount it + borg mount "${REPOSITORY}" "${MOUNT_PATH}"; +else + echo "Unmounting ${MOUNT_PATH}"; + # will fail with error if not mounted, but not critical + borg umount "${MOUNT_PATH}"; +fi; + +## END