Borg Backup Wrapper Scripts

This commit is contained in:
Clemens Schwaighofer
2021-12-13 10:36:44 +09:00
commit 93f8ea6054
12 changed files with 1571 additions and 0 deletions

13
Readme.md Normal file
View File

@@ -0,0 +1,13 @@
# Borg Backup Wrapper Scripts
These scripts are wrappers around the main borg backup scripts.
Modules for plain file backup, mysql and postgresql backup exists.
## Basic Settings
## File backup settings
## PostgreSQL backup settings
## MySQL backup settings

188
borg.backup.file.sh Executable file
View File

@@ -0,0 +1,188 @@
#!/usr/bin/env bash
# Run -I first to initialize repository
# There are no automatic repository checks unless -C is given
# set last edit date + time
MODULE="file";
MODULE_VERSION="0.1.0";
DIR="${BASH_SOURCE%/*}"
if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
. "${DIR}/borg.backup.functions.init.sh";
# include and exclude file
INCLUDE_FILE="borg.backup.file.include";
EXCLUDE_FILE="borg.backup.file.exclude";
BACKUP_INIT_CHECK="borg.backup.file.init";
. "${DIR}/borg.backup.functions.check.sh";
# exit if include file is missing
if [ ! -f "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
echo "[! $(date +'%F %T')] The include folder file ${INCLUDE_FILE} is missing";
exit 1;
fi;
# folders to backup
FOLDERS=();
# this if for debug output with quoted folders
FOLDERS_Q=();
# include list
while read include_folder; do
# strip any leading spaces from that folder
include_folder=$(echo "${include_folder}" | sed -e 's/^[ \t]*//');
# check that those folders exist, warn on error,
# but do not exit unless there are no valid folders at all
# skip folders that are with # in front (comment)
if [[ "${include_folder}" =~ ${REGEX_COMMENT} ]]; then
echo "# [C] Comment: '${include_folder}'";
else
# skip if it is empty
if [ ! -z "${include_folder}" ]; then
# if this is a glob, do a double check that the base folder actually exists (?)
if [[ "${include_folder}" =~ $REGEX_GLOB ]]; then
# if this is */ then allow it
# remove last element beyond the last /
# if there is no path, just allow it (general rule)
_include_folder=${include_folder%/*};
# if still a * inside -> add as is, else check for folder
if [[ "${include_folder}" =~ $REGEX_GLOB ]]; then
FOLDER_OK=1;
echo "+ [I] Backup folder with folder path glob '${include_folder}'";
# glob (*) would be escape so we replace it with a temp part and then reinsert it
FOLDERS_Q+=($(printf "%q" "$(echo "${include_folder}" | sed 's/\*/_STARGLOB_/g')" | sed 's/_STARGLOB_/\*/g'));
FOLDERS+=("${include_folder}");
elif [ ! -d "${_include_folder}" ]; then
echo "- [I] Backup folder with glob '${include_folder}' does not exist or is not accessable";
else
FOLDER_OK=1;
echo "+ [I] Backup folder with glob '${include_folder}'";
# we need glob fix
FOLDERS_Q+=($(printf "%q" "$(echo "${include_folder}" | sed 's/\*/_STARGLOB_/g')" | sed 's/_STARGLOB_/\*/g'));
FOLDERS+=("${include_folder}");
fi;
# normal folder
elif [ ! -d "${include_folder}" ] && [ ! -e "${include_folder}" ]; then
echo "- [I] Backup folder or file '${include_folder}' does not exist or is not accessable";
else
FOLDER_OK=1;
# if it is a folder, remove the last / or the symlink check will not work
if [ -d "${include_folder}" ]; then
_include_folder=${include_folder%/*};
else
_include_folder=${include_folder};
fi;
# Warn if symlink & folder -> only smylink will be backed up
if [ -h "${_include_folder}" ]; then
echo "~ [I] Target '${include_folder}' is a symbolic link. No real data will be backed up";
else
echo "+ [I] Backup folder or file '${include_folder}'";
fi;
FOLDERS_Q+=($(printf "%q" "${include_folder}"));
FOLDERS+=("${include_folder}");
fi;
fi;
fi;
done<"${BASE_FOLDER}${INCLUDE_FILE}";
# exclude list
if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
# check that the folders in that exclude file are actually valid,
# remove non valid ones and warn
#TMP_EXCLUDE_FILE=$(mktemp --tmpdir ${EXCLUDE_FILE}.XXXXXXXX); # non mac
TMP_EXCLUDE_FILE=$(mktemp "${TEMPDIR}${EXCLUDE_FILE}".XXXXXXXX);
while read exclude_folder; do
# strip any leading spaces from that folder
exclude_folder=$(echo "${exclude_folder}" | sed -e 's/^[ \t]*//');
# folder or any type of file is ok
# because of glob files etc, exclude only comments (# start)
if [[ "${exclude_folder}" =~ ${REGEX_COMMENT} ]]; then
echo "# [C] Comment: '${exclude_folder}'";
else
# skip if it is empty
if [ ! -z "${exclude_folder}" ]; then
# if it DOES NOT start with a / we assume free folder and add as is
if [[ "${exclude_folder}" != /* ]]; then
echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE};
echo "+ [E] General exclude: '${exclude_folder}'";
# if this is a glob, do a double check that the base folder actually exists (?)
elif [[ "${exclude_folder}" =~ $REGEX_GLOB ]]; then
# remove last element beyond the last /
# if there is no path, just allow it (general rule)
_exclude_folder=${exclude_folder%/*};
if [ ! -d "${_exclude_folder}" ]; then
echo "- [E] Exclude folder with glob '${exclude_folder}' does not exist or is not accessable";
else
echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE};
echo "+ [E] Exclude folder with glob '${exclude_folder}'";
fi;
# do a warning for a possible invalid folder
# but we do not a exclude if the data does not exist
elif [ ! -d "${exclude_folder}" ] && [ ! -e "${exclude_folder}" ]; then
echo "- [E] Exclude folder or file '${exclude_folder}' does not exist or is not accessable";
else
echo "${exclude_folder}" >> ${TMP_EXCLUDE_FILE};
# if it is a folder, remove the last / or the symlink check will not work
if [ -d "${exclude_folder}" ]; then
_exclude_folder=${exclude_folder%/*};
else
_exclude_folder=${exclude_folder};
fi;
# warn if target is symlink folder
if [ -h "${_exclude_folder}" ]; then
echo "~ [I] Target '${exclude_folder}' is a symbolic link. No real data will be excluded from backup";
else
echo "+ [E] Exclude folder or file '${exclude_folder}'";
fi;
fi;
fi;
fi;
done<"${BASE_FOLDER}${EXCLUDE_FILE}";
# avoid blank file add by checking if the tmp file has a size >0
if [ -s "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
OPT_EXCLUDE="--exclude-from=${TMP_EXCLUDE_FILE}";
fi;
fi;
# add the repository set before we add the folders
# base command
COMMAND="borg create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} ";
# add repoistory, after that the folders will be added on call
COMMAND=${COMMAND}${REPOSITORY}::${BACKUP_SET};
# if info print info and then abort run
. "${DIR}/borg.backup.functions.info.sh";
if [ $FOLDER_OK -eq 1 ]; then
echo "--- [BACKUP: $(date +'%F %T')] ------------------------------------------->";
# show command
if [ ${DEBUG} -eq 1 ]; then
echo $(echo ${COMMAND} | sed -e 's/[ ][ ]*/ /g') ${FOLDERS_Q[*]};
fi;
# execute backup command
if [ ${DRYRUN} -eq 0 ]; then
# need to redirect std error to std out so all data is printed to the correct pipe
# for the IFS="#" to work we need to replace options spaces with exactly ONE #
$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') ${FOLDERS[*]} 2>&1 || echo "[!] Borg backup aborted.";
fi;
# remove the temporary exclude file if it exists
if [ -f "${TMP_EXCLUDE_FILE}" ]; then
rm -f "${TMP_EXCLUDE_FILE}";
fi;
else
echo "[! $(date +'%F %T')] No folders where set for the backup";
exit 1;
fi;
# clean up, always verbose
echo "--- [PRUNE : $(date +'%F %T')] ------------------------------------------->";
# build command
COMMAND="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} ${KEEP_OPTIONS[*]} ${REPOSITORY}";
echo "Prune repository with keep${KEEP_INFO:1}";
if [ ${DEBUG} -eq 1 ]; then
echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g';
fi;
# for the IFS="#" to work we need to replace options spaces with exactly ONE #
$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted";
. "${DIR}/borg.backup.functions.close.sh";
# __END__

View File

@@ -0,0 +1,321 @@
#!/usr/bin/env bash
# start time in seconds
START=$(date +'%s');
# start logging from here
exec &> >(tee -a "${LOG}");
echo "=== [START : $(date +'%F %T')] ==[${MODULE}]====================================>";
# show info for version always
echo "Script version: ${VERSION}";
# show type
echo "Backup module : ${MODULE}";
echo "Module version: ${MODULE_VERSION}";
# show base folder always
echo "Base folder : ${BASE_FOLDER}";
# if force check is true set CHECK to 1unless INFO is 1
# Needs bash 4.0 at lesat for this
if [ "${FORCE_CHECK,,}" = "true" ] && [ ${INFO} -eq 0 ]; then
CHECK=1;
if [ ${DEBUG} -eq 1 ]; then
echo "Force repository check";
fi;
fi;
# remote borg path
if [ ! -z "${TARGET_BORG_PATH}" ]; then
if [[ "${TARGET_BORG_PATH}" =~ \ |\' ]]; then
echo "Space found in ${TARGET_BORG_PATH}. Aborting";
echo "There are issues with passing on paths with spaces"
echo "as parameters"
exit;
fi;
OPT_REMOTE="--remote-path="$(printf "%q" "${TARGET_BORG_PATH}");
fi;
if [ -z "${TARGET_FOLDER}" ]; then
echo "[! $(date +'%F %T')] No target folder has been set yet";
exit 1;
else
# There are big issues with TARGET FOLDERS with spaces
# we should abort anything with this
if [[ "${TARGET_FOLDER}" =~ \ |\' ]]; then
echo "Space found in ${TARGET_FOLDER}. Aborting";
echo "There is some problem with passing paths with spaces as";
echo "repository base folder"
exit;
fi;
# This does not care for multiple trailing or leading slashes
# it just makes sure we have at least one set
# for if we have a single slash, remove it
TARGET_FOLDER=${TARGET_FOLDER%/}
TARGET_FOLDER=${TARGET_FOLDER#/}
# and add slash front and back and escape the path
TARGET_FOLDER=$(printf "%q" "/${TARGET_FOLDER}/");
fi;
# if we have user/host then we build the ssh command
TARGET_SERVER='';
# allow host only (if full setup in .ssh/config)
# user@host OR ssh://user@host:port/ IF TARGET_PORT is set
# user/host/port
if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then
TARGET_SERVER="ssh://${TARGET_USER}@${TARGET_HOST}:${TARGET_PORT}/";
# host/port
elif [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then
TARGET_SERVER="ssh://${TARGET_HOST}:${TARGET_PORT}/";
# user/host
elif [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then
TARGET_SERVER="${TARGET_USER}@${TARGET_HOST}:";
# host
elif [ ! -z "${TARGET_HOST}" ]; then
TARGET_SERVER="${TARGET_HOST}:";
fi;
# we dont allow special characters, so we don't need to special escape it
REPOSITORY="${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}";
echo "Repository : ${REPOSITORY}";
# check compression if given is valid and check compression level is valid if given
if [ ! -z "${COMPRESSION}" ]; then
# valid compression
if [ "${COMPRESSION}" = "lz4" ] || [ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ] || [ "${COMPRESSION}" = "zstd" ]; then
OPT_COMPRESSION="-C=${COMPRESSION}";
# if COMPRESSION_LEVEL, check it is a valid regex
# for zlib, zstd, lzma
if [ ! -z "${COMPRESSION_LEVEL}" ] && ([ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ] || [ "${COMPRESSION}" = "zstd" ]); then
MIN_COMPRESSION=0;
MAX_COMPRESSION=0;
case "${COMPRESSION}" in
zlib|lzma)
MIN_COMPRESSION=0;
MAX_COMPRESSION=9;
;;
zstd)
MIN_COMPRESSION=1;
MAX_COMPRESSION=22;
;;
*)
MIN_COMPRESSION=0;
MAX_COMPRESSION=0;
;;
esac;
# if [ "${COMPRESSION}" = "zlib" ] || [ "${COMPRESSION}" = "lzma" ]
# MIN_COMPRESSION=0;
# MAX_COMPRESSION=9;
# elif [ "${COMPRESSION}" = "zstd" ]; then
# MIN_COMPRESSION=1;
# MAX_COMPRESSION=22;
# fi;
error_message="[! $(date +'%F %T')] Compression level for ${COMPRESSION} needs to be a numeric value between ${MIN_COMPRESSION} and ${MAX_COMPRESSION}: ${COMPRESSION_LEVEL}";
if ! [[ "${COMPRESSION_LEVEL}" =~ ${REGEX_NUMERIC} ]]; then
echo ${error_message};
exit 1;
elif [ ${COMPRESSION_LEVEL} -lt ${MIN_COMPRESSION} ] || [ ${COMPRESSION_LEVEL} -gt ${MAX_COMPRESSION} ]; then
echo ${error_message};
exit 1;
else
OPT_COMPRESSION=${OPT_COMPRESSION}","${COMPRESSION_LEVEL};
fi;
fi;
else
echo "[! $(date +'%F %T')] Compress setting need to be lz4, zstd, zlib or lzma. Or empty for no compression: ${COMPRESSION}";
exit 1;
fi;
fi;
# home folder, needs to be set if there is eg a HOME=/ in the crontab
if [ ! -w "${HOME}" ] || [ "${HOME}" = '/' ]; then
HOME=$(eval echo "$(whoami)");
fi;
# build options and info string,
# also flag BACKUP_SET check if hourly is set
KEEP_OPTIONS=();
KEEP_INFO="";
BACKUP_SET_CHECK=0;
if [ ${KEEP_LAST} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}");
KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}";
fi;
if [ ${KEEP_HOURS} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}");
KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}";
BACKUP_SET_CHECK=1;
fi;
if [ ${KEEP_DAYS} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}");
KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}";
fi;
if [ ${KEEP_WEEKS} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}");
KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}";
fi;
if [ ${KEEP_MONTHS} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}");
KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}";
fi;
if [ ${KEEP_YEARS} -gt 0 ]; then
KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}");
KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}";
fi;
if [ ! -z "${KEEP_WITHIN}" ]; then
# check for invalid string. can only be number + H|d|w|m|y
if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then
KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}");
KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}";
if [[ "${KEEP_WITHIN}" == *"H"* ]]; then
BACKUP_SET_CHECK=1;
fi;
else
echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string.";
exit 1;
fi;
fi;
# abort if KEEP_OPTIONS is empty
if [ -z "${KEEP_OPTIONS}" ]; then
echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format.";
exit 1;
fi;
# set BACKUP_SET if empty, check for for DATE is set
if [ -z "${BACKUP_SET}" ]; then
# DATE is deprecated and will be removed
if [ ! -z "${DATE}" ]; then
echo "[!] DEPRECATED: The use of DATE variable is deprecated, use BACKUP_SET instead";
BACKUP_SET="${DATE}";
else
# default
BACKUP_SET="{now:%Y-%m-%d}";
fi;
fi;
# backup set check, and there is no hour entry (%H) in the archive string
# we add T%H:%M:%S in this case, before the last }
if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then
BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/");
fi;
# for folders list split set to "#" and keep the old setting as is
_IFS=${IFS};
IFS="#";
# turn off for non file
if [ "${MODULE}" != "file" ]; then
IFS=${_IFS};
fi;
# general borg settings
# set base path to config directory to keep cache/config separated
export BORG_BASE_DIR="${BASE_FOLDER}";
# ignore non encrypted access
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=${_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK};
# ignore moved repo access
export BORG_RELOCATED_REPO_ACCESS_IS_OK=${_BORG_RELOCATED_REPO_ACCESS_IS_OK};
# and for debug print that tout
if [ ${DEBUG} -eq 1 ]; then
echo "export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=${_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK};";
echo "export BORG_RELOCATED_REPO_ACCESS_IS_OK=${_BORG_RELOCATED_REPO_ACCESS_IS_OK};";
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
fi;
# prepare debug commands only
COMMAND_EXPORT="export BORG_BASE_DIR=\"${BASE_FOLDER}\";"
COMMAND_INFO="${COMMAND_EXPORT}borg info ${OPT_REMOTE} ${REPOSITORY}";
# if the is not there, call init to create it
# if this is user@host, we need to use ssh command to check if the file is there
# else a normal check is ok
# unless explicit given, check is skipped
if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then
echo "--- [CHECK : $(date +'%F %T')] ------------------------------------------->";
if [ ! -z "${TARGET_SERVER}" ]; then
if [ ${DEBUG} -eq 1 ]; then
echo "borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\"";
fi;
# use borg info and check if it returns "Repository ID:" in the first line
REPO_CHECK=$(borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:");
# this is currently a hack to work round the error code in borg info
# this checks if REPO_CHECK holds this error message and then starts init
regex="^Some part of the script failed with an error:";
if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${regex} ]]; then
INIT_REPOSITORY=1;
fi;
elif [ ! -d "${REPOSITORY}" ]; then
INIT_REPOSITORY=1;
fi;
# if check but no init and repo is there but init file is missing set it
if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 0 ] &&
[ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then
# write init file
echo "[!] Add missing init check file";
echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}";
fi;
# end if checked but repository is not here
if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then
echo "[! $(date +'%F %T')] No repository. Please run with -I flag to initialze repository";
exit 1;
fi;
if [ ${EXIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ]; then
echo "Repository exists";
echo "For more information run:"
echo "${COMMAND_INFO}";
echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>";
exit;
fi;
fi;
if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then
echo "--- [INIT : $(date +'%F %T')] ------------------------------------------->";
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}";
fi
if [ ${DRYRUN} -eq 0 ]; then
# should trap and exit properly here
borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY};
# write init file
echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}";
echo "Repository initialized";
echo "For more information run:"
echo "${COMMAND_INFO}";
fi
echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>";
# exit after init
exit;
elif [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 0 ]; then
echo "[! $(date +'%F %T')] Repository already initialized";
echo "For more information run:"
echo "${COMMAND_INFO}";
exit 1;
fi;
# check for init file
if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then
echo "[! $(date +'%F %T')] It seems the repository has never been initialized."
echo "Please run -I to initialize or if already initialzed run with -C for init update."
exit 1;
fi;
# PRINT OUT current data, only do this if REPO exists
if [ ${PRINT} -eq 1 ]; then
echo "--- [PRINT : $(date +'%F %T')] ------------------------------------------->";
FORMAT="{archive} {comment:6} {start} - {end} [{id}] ({username}@{hostname}){NL}"
# show command on debug or dry run
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}";
fi;
# run info command if not a dry drun
if [ ${DRYRUN} -eq 0 ]; then
borg list ${OPT_REMOTE} --format "${FORMAT}" ${REPOSITORY} ;
fi;
if [ ${VERBOSE} -eq 1 ]; then
echo "";
echo "Base command info:"
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]";
echo "Replace [COMMAND] with list for listing or extract for restoring backup data."
echo "Replace [BACKUP] with archive name."
echo "If no [PATH] is given then all files will be restored."
echo "Before extracting -n (dry run) is recommended to use."
echo "If archive size is needed the info command with archive name has to be used."
echo "When listing (list) data the --format command can be used."
echo "Example: \"{mode} {user:6} {group:6} {size:8d} {csize:8d} {dsize:8d} {dcsize:8d} {mtime} {path}{extra} [{health}]{NL}\""
else
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]";
fi;
exit;
fi;
# __END__

View File

@@ -0,0 +1,9 @@
#!/usr/bin/env bash
# unset borg settings
unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK
DURATION=$[ $(date +'%s')-$START ];
echo "=== [Run time: $(convert_time ${DURATION})]";
echo "=== [END : $(date +'%F %T')] ==[${MODULE}]===================>";
# __END__

View File

@@ -0,0 +1,30 @@
#!/usr/bin/env bash
if [ ${INFO} -eq 1 ]; then
echo "--- [INFO : $(date +'%F %T')] ------------------------------------------->";
# show command on debug or dry run
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg info ${OPT_REMOTE} ${REPOSITORY}";
fi;
# run info command if not a dry drun
if [ ${DRYRUN} -eq 0 ]; then
borg info ${OPT_REMOTE} ${REPOSITORY};
fi;
if [ "${MODULE}" = "files" ]; then
if [ $FOLDER_OK -eq 1 ]; then
echo "--- [Run command]:";
#IFS="#";
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${COMMAND} "${FOLDERS_Q[*]};
else
echo "[!] No folders where set for the backup";
fi;
# remove the temporary exclude file if it exists
if [ -f "${TMP_EXCLUDE_FILE}" ]; then
rm -f "${TMP_EXCLUDE_FILE}";
fi;
fi;
echo "=== [END : $(date +'%F %T')] ==[${MODULE}]====================================>";
exit;
fi;
# __END__

View File

@@ -0,0 +1,395 @@
#!/usr/bin/env bash
set -ETu #-e -o pipefail
trap cleanup SIGINT SIGTERM ERR
cleanup() {
# script cleanup here
echo "Some part of the script failed with an error: $? @LINE: $(caller)";
# unset exported vars
unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK;
# end trap
trap - SIGINT SIGTERM ERR
}
# on exit unset any exported var
trap "unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK" EXIT;
# version for all general files
VERSION="3.0.0";
# default log folder if none are set in config or option
_LOG_FOLDER="/var/log/borg.backup/";
# log file name is set based on BACKUP_FILE, .log is added
LOG_FOLDER="";
# should be there on everything
TEMPDIR="/tmp/";
# creates borg backup based on the include/exclude files
# if base borg folder (backup files) does not exist, it will automatically init it
# base folder
BASE_FOLDER="/usr/local/scripts/borg/";
# base settings and init flag
SETTINGS_FILE="borg.backup.settings";
# include files
INCLUDE_FILE="";
EXCLUDE_FILE="";
# backup folder initialzed check
BACKUP_INIT_CHECK="";
# debug/verbose
VERBOSE=0;
LIST=0;
DEBUG=0;
DRYRUN=0;
INFO=0;
CHECK=0;
INIT=0;
EXIT=0;
PRINT=0;
# flags, set to no to disable
_BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK="yes";
_BORG_RELOCATED_REPO_ACCESS_IS_OK="yes";
# other variables
TARGET_SERVER="";
REGEX="";
REGEX_COMMENT="^[\ \t]*#";
REGEX_GLOB='\*';
REGEX_NUMERIC="^[0-9]{1,2}$";
PRUNE_DEBUG="";
INIT_REPOSITORY=0;
FOLDER_OK=0;
TMP_EXCLUDE_FILE="";
# opt flags
OPT_VERBOSE="";
OPT_PROGRESS="";
OPT_LIST="";
OPT_REMOTE="";
OPT_LOG_FOLDER="";
OPT_EXCLUDE="";
# config variables (will be overwritten from .settings file)
TARGET_USER="";
TARGET_HOST="";
TARGET_PORT="";
TARGET_BORG_PATH="";
TARGET_FOLDER="";
BACKUP_FILE="";
SUB_BACKUP_FILE="";
# lz4, zstd 1-22 (3), zlib 0-9 (6), lzma 0-9 (6)
COMPRESSION="zstd";
COMPRESSION_LEVEL=3;
SUB_COMPRESSION="";
SUB_COMPRESSION_LEVEL="";
# encryption settings
ENCRYPTION="none";
# force check always
FORCE_CHECK="false";
DATE=""; # to be deprecated
BACKUP_SET="";
SUB_BACKUP_SET="";
# for database backup only
DATABASE_FULL_DUMP="";
DATABASE_USER="";
# only for mysql old config file
MYSQL_DB_CONFIG="";
MYSQL_DB_CONFIG_PARAM="";
# default keep 7 days, 4 weeks, 6 months
# if set 0, ignore
# note that for last/hourly it is needed to create a different
# BACKUP SET that includes hour and minute information
# IF BACKUP_SET is empty, this is automatically added
# general keep last, if only this is set only last n will be kept
KEEP_LAST=0;
KEEP_HOURS=0;
KEEP_DAYS=7;
KEEP_WEEKS=4;
KEEP_MONTHS=6;
KEEP_YEARS=1;
# in the format of nY|M|d|h|m|s
KEEP_WITHIN="";
# sub override init to empty
SUB_KEEP_LAST="";
SUB_KEEP_HOURS="";
SUB_KEEP_DAYS="";
SUB_KEEP_WEEKS="";
SUB_KEEP_MONTHS="";
SUB_KEEP_YEARS="";
SUB_KEEP_WITHIN="";
function usage()
{
cat <<- EOT
Usage: ${0##/*/} [-c <config folder>] [-v] [-d]
-c <config folder>: if this is not given, ${BASE_FOLDER} is used
-L <log folder>: override config set and default log folder
-P: print list of archives created
-C: check if repository exists, if not abort
-E: exit after check
-I: init repository (must be run first)
-v: be verbose
-i: print out only info
-l: list files during backup
-d: debug output all commands
-n: only do dry run
-h: this help page
Version : ${VERSION}
Module Version: ${MODULE_VERSION}
Module : ${MODULE}
EOT
}
# set options
while getopts ":c:L:vldniCEIPh" opt; do
case "${opt}" in
c|config)
BASE_FOLDER=${OPTARG};
;;
L|log)
OPT_LOG_FOLDER=${OPTARG};
;;
C|Check)
# will check if repo is there and abort if not
CHECK=1;
;;
E|Exit)
# exit after check
EXIT=1;
;;
I|Init)
# will check if there is a repo and init it
# previoous this was default
CHECK=1;
INIT=1;
;;
P|Print)
# use borg list to print list of archves
PRINT=1;
;;
v|verbose)
VERBOSE=1;
;;
l|list)
LIST=1;
;;
i|info)
INFO=1;
;;
d|debug)
DEBUG=1;
;;
n|dryrun)
DRYRUN=1;
;;
h|help)
usage;
exit;
;;
:)
echo "Option -$OPTARG requires an argument."
;;
\?)
echo -e "\n Option does not exist: ${OPTARG}\n";
usage;
exit 1;
;;
esac;
done;
# add trailing slasd for base folder
[[ "${BASE_FOLDER}" != */ ]] && BASE_FOLDER=${BASE_FOLDER}"/";
# must have settings file there, if not, abort early
if [ ! -f "${BASE_FOLDER}${SETTINGS_FILE}" ]; then
echo "No settings file could be found: ${BASE_FOLDER}${SETTINGS_FILE}";
exit 1;
fi;
if [ ! -w "${BASE_FOLDER}" ]; then
echo "Cannot write to BASE_FOLDER ${BASE_FOLDER}";
echo "Is the group set to 'backup' and is this group allowed to write?"
echo "chgrp -R backup ${BASE_FOLDER}";
echo "chmod -R g+rws ${BASE_FOLDER}";
exit 1;
fi;
# info -i && -C/-I cannot be run together
if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${INFO} -eq 1 ]; then
echo "Cannot have -i info option and -C check or -I initialize option at the same time";
exit 1;
fi;
# print -P cannot be run with -i/-C/-I together
if [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INFO} -eq 1 ]; then
echo "Cannot have -P print option and -i info, -C check or -I initizalize option at the same time";
exit 1;
fi;
# verbose & progress
if [ ${VERBOSE} -eq 1 ]; then
OPT_VERBOSE="-v";
OPT_PROGRESS="-p";
fi;
# list files
if [ ${LIST} -eq 1 ]; then
OPT_LIST="--list";
fi;
if [ ${DRYRUN} -eq 1 ]; then
PRUNE_DEBUG="--dry-run";
fi;
# read config file
. "${BASE_FOLDER}${SETTINGS_FILE}";
# ** SUB LOAD
# a settings file always end in .settings, replace that with lower case MODULE.settings
SETTINGS_FILE_SUB=$(echo "${SETTINGS_FILE}" | sed -e "s/\.settings/\.${MODULE,,}\.settings/");
# if mysql/pgsql run, load sub settings
if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then
. "${BASE_FOLDER}${SETTINGS_FILE_SUB}";
# if SUB_ set override master
if [ ! -z "${SUB_BACKUP_FILE}" ]; then
BACKUP_FILE=${SUB_BACKUP_FILE}
fi;
# add module name to backup file, always
BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg};
# if sub backup set it set, override current
if [ ! -z "${SUB_BACKUP_SET}" ]; then
BACKUP_SET=${SUB_BACKUP_SET};
fi;
# ovrride compression
if [ ! -z "${SUB_COMPRESSION}" ]; then
COMPRESSION=${SUB_COMPRESSION};
fi;
if [ ! -z "${SUB_COMPRESSION_LEVEL}" ]; then
COMPRESSION_LEVEL=${SUB_COMPRESSION_LEVEL};
fi;
# check override for keep time
if [ ! -z "${SUB_KEEP_LAST}" ]; then
KEEP_LAST=${SUB_KEEP_LAST};
fi;
if [ ! -z "${SUB_KEEP_HOURS}" ]; then
KEEP_HOURS=${SUB_KEEP_HOURS};
fi;
if [ ! -z "${SUB_KEEP_DAYS}" ]; then
KEEP_DAYS=${SUB_KEEP_DAYS};
fi;
if [ ! -z "${SUB_KEEP_WEEKS}" ]; then
KEEP_WEEKS=${SUB_KEEP_WEEKS};
fi;
if [ ! -z "${SUB_KEEP_YEARS}" ]; then
KEEP_YEARS=${SUB_KEEP_YEARS};
fi;
if [ ! -z "${SUB_KEEP_LAST}" ]; then
KEEP_LAST=${SUB_KEEP_LAST};
fi;
if [ ! -z "${SUB_KEEP_WITHIN}" ]; then
KEEP_WITHIN=${SUB_KEEP_WITHIN};
fi;
fi;
# backup file must be set
if [ -z "${BACKUP_FILE}" ]; then
echo "No BACKUP_FILE set";
exit;
fi;
# backup file (folder) must end as .borg
# BACKUP FILE also cannot start with / or have / inside or start with ~
# valid file name check, alphanumeric, -,._ ...
if ! [[ "${BACKUP_FILE}" =~ ^[A-Za-z0-9,._-]+\.borg$ ]]; then
echo "BACKUP_FILE ${BACKUP_FILE} can only contain A-Z a-z 0-9 , . _ - chracters and must end with .borg";
exit 1;
fi;
# error if the repository file still has the default name
# This is just for old sets
REGEX="^some\-prefix\-";
if [[ "${BACKUP_FILE}" =~ ${REGEX} ]]; then
echo "[DEPRECATED] The repository name still has the default prefix: ${BACKUP_FILE}";
exit 1;
fi;
# check LOG_FOLDER, TARGET_BORG_PATH, TARGET_FOLDER must have no ~/ as start position
if [[ ${LOG_FOLDER} =~ ^~\/ ]]; then
echo "LOG_FOLDER path cannot start with ~/. Path must be absolute: ${LOG_FOLDER}";
exit 1;
fi;
if [[ ${TARGET_BORG_PATH} =~ ^~\/ ]]; then
echo "TARGET_BORG_PATH path cannot start with ~/. Path must be absolute: ${TARGET_BORG_PATH}";
exit 1;
fi;
if [[ ${TARGET_FOLDER} =~ ^~\/ ]]; then
echo "TARGET_FOLDER path cannot start with ~/. Path must be absolute: ${TARGET_FOLDER}";
exit 1;
fi
# log file set and check
# option folder overrides all other folders
if [ ! -z "${OPT_LOG_FOLDER}" ]; then
LOG_FOLDER="${OPT_LOG_FOLDER}";
fi;
# if empty folder set to default folder
if [ -z "${LOG_FOLDER}" ]; then
LOG_FOLDER="${_LOG_FOLDER}";
fi;
# if folder does not exists create it
if [ ! -d "${LOG_FOLDER}" ]; then
mkdir "${LOG_FOLDER}";
fi;
# set the output log folder
# LOG=$(printf "%q" "${LOG_FOLDER}/${BACKUP_FILE}.log");
LOG="${LOG_FOLDER}/${BACKUP_FILE}.log";
# fail if not writeable to folder or file
if [[ -f "${LOG}" && ! -w "${LOG}" ]] || [[ ! -f "${LOG}" && ! -w "${LOG_FOLDER}" ]]; then
echo "Log folder or log file is not writeable: ${LOG}";
echo "Is the group set to 'backup' and is this group allowed to write?"
echo "chgrp -R backup ${LOG}";
echo "chmod -R g+rws ${LOG}";
exit 1;
fi;
# if ENCRYPTION is empty or not in the valid list fall back to none
if [ -z "${ENCRYPTION}" ]; then
ENCRYPTION="none";
#else
# TODO check for invalid encryption string
fi;
## FUNCTIONS
# METHOD: convert_time
# PARAMS: timestamp in seconds or with milliseconds (nnnn.nnnn)
# RETURN: formated string with human readable time (d/h/m/s)
# CALL : var=$(convert_time $timestamp);
# DESC : converts a timestamp or a timestamp with float milliseconds
# to a human readable format
# output is in days/hours/minutes/seconds
function convert_time
{
timestamp=${1};
# round to four digits for ms
timestamp=$(printf "%1.4f" $timestamp);
# get the ms part and remove any leading 0
ms=$(echo ${timestamp} | cut -d "." -f 2 | sed -e 's/^0*//');
timestamp=$(echo ${timestamp} | cut -d "." -f 1);
timegroups=(86400 3600 60 1); # day, hour, min, sec
timenames=("d" "h" "m" "s"); # day, hour, min, sec
output=( );
time_string=;
for timeslice in ${timegroups[@]}; do
# floor for the division, push to output
output[${#output[*]}]=$(awk "BEGIN {printf \"%d\", ${timestamp}/${timeslice}}");
timestamp=$(awk "BEGIN {printf \"%d\", ${timestamp}%${timeslice}}");
done;
for ((i=0; i<${#output[@]}; i++)); do
if [ ${output[$i]} -gt 0 ] || [ ! -z "$time_string" ]; then
if [ ! -z "${time_string}" ]; then
time_string=${time_string}" ";
fi;
time_string=${time_string}${output[$i]}${timenames[$i]};
fi;
done;
if [ ! -z ${ms} ] && [ ${ms} -gt 0 ]; then
time_string=${time_string}" "${ms}"ms";
fi;
# just in case the time is 0
if [ -z "${time_string}" ]; then
time_string="0s";
fi;
echo -n "${time_string}";
}
# __END__

View File

@@ -0,0 +1,11 @@
# Borg backup wrapper scripts settings
# override settings in borg.backup.settings with SUB_ prefix
# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_*
# set to 1 to dump all into one file instead of per database
# note that with this databases that have been dropped need to be pruned manually
# if 'schema' word is used, only schema data is dumped
DATABASE_FULL_DUMP="";
# db config (for older mysql setups without ident basedlogin)
MYSQL_DB_CONFIG="";

209
borg.backup.mysql.sh Executable file
View File

@@ -0,0 +1,209 @@
#!/usr/bin/env bash
# Backup MySQL/MariaDB
# default is per table dump, can be set to one full dump
# config override set in borg.backup.mysql.settings
# if run as mysql user, be sure user is in the backup group
# Run -I first to initialize repository
# There are no automatic repository checks unless -C is given
# set last edit date + time
MODULE="mysql"
MODULE_VERSION="1.0.0";
DIR="${BASH_SOURCE%/*}"
if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
# init system
. "${DIR}/borg.backup.functions.init.sh";
# include and exclude file
INCLUDE_FILE="borg.backup.mysql.include";
EXCLUDE_FILE="borg.backup.mysql.exclude";
SCHEMA_ONLY_FILE="borg.backup.mysql.schema-only";
BACKUP_INIT_CHECK="borg.backup.mysql.init";
# check valid data
. "${DIR}/borg.backup.functions.check.sh";
# if info print info and then abort run
. "${DIR}/borg.backup.functions.info.sh";
# if there is an DB extra config
# on current installs there should be a root or mysql user with unix socket connection
# the script should by run as the mysql or root user (sudo -u mysql ...)
if [ -f "${MYSQL_DB_CONFIG}" ]; then
# MYSQL_DB_CONFIG='/root/.my.cnf';
MYSQL_DB_CONFIG_PARAM="--defaults-extra-file=${MYSQL_DB_CONFIG}";
fi;
MYSQL_BASE_PATH='/usr/bin/';
MYSQL_DUMP=${MYSQL_BASE_PATH}'mysqldump';
MYSQL_CMD=${MYSQL_BASE_PATH}'mysql';
# no dump or mysql, bail
if [ ! -f "${MYSQL_DUMP}" ]; then
echo "[! $(date +'%F %T')] mysqldump binary not found";
exit 1;
fi;
if [ ! -f "${MYSQL_CMD}" ]; then
echo "[! $(date +'%F %T')] mysql binary not found";
exit 1;
fi;
# check that the user can actually do, else abort here
# note: this is the only way to not error
_MYSQL_CHECK=$(mysqladmin ${MYSQL_DB_CONFIG_PARAM} ping 2>&1);
_MYSQL_OK=$(echo "${_MYSQL_CHECK}" | grep "is alive");
if [ -z "${_MYSQL_OK}" ]; then
echo "[! $(date +'%F %T')] Current user has no access right to mysql database";
exit 1;
fi;
# below is for file name only
# set DB_VERSION (Distrib n.n.n-type)
# NEW: mysql Ver 15.1 Distrib 10.5.12-MariaDB, for debian-linux-gnu (x86_64) using EditLine wrapper
# OLD: mysql Ver 14.14 Distrib 5.7.35, for Linux (x86_64) using EditLine wrapper
_DB_VERSION_TYPE=$("${MYSQL_CMD}" --version);
_DB_VERSION=$(echo "${_DB_VERSION_TYPE}" | sed 's/.*Distrib \([0-9]\{1,\}\.[0-9]\{1,\}\)\.[0-9]\{1,\}.*/\1/');
DB_VERSION=$(echo "${_DB_VERSION}" | cut -d " " -f 1);
# temporary until correct type detection is set
DB_TYPE="mysql";
# try to get type from -string, if empty set mysql
# if [[ ${_DB_VERSION_TYPE##*-*} ]]; then
# DB_TYPE="mysql";
# else
# DB_TYPE=$(echo "${_DB_TYPE}" | sed -e 's/.*[0-9]-\([A-Za-z]\{1,\}\).*/\1/');
# fi;
DB_PORT='3306';
DB_HOST='local';
# those dbs have to be dropped with skip locks (single transaction)
NOLOCKDB="information_schema performance_schema"
NOLOCKS="--single-transaction"
# those tables need to be dropped with EVENTS
EVENTDB="mysql"
EVENTS="--events"
# borg call, replace ##...## parts
_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
# ALL IN ONE FILE or PER DATABASE FLAG
if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
SCHEMA_ONLY='';
schema_flag='data';
if [ "${DATABASE_FULL_DUMP}" = "schema" ]; then
SCHEMA_ONLY='--no-data';
schema_flag='schema';
fi;
echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
# We only do a full backup and not per table backup here
# Filename
FILENAME="all-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql"
# backup set:
BACKUP_SET_NAME="all-${schema_flag}-${BACKUP_SET}";
BACKUP_SET_PREFIX="all-";
# borg call
BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}";
echo "${BORG_PRUNE}";
fi;
if [ ${DRYRUN} -eq 0 ]; then
${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL};
_backup_error=$?;
if [ $_backup_error -ne 0 ]; then
echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
exit $_backup_error;
fi;
fi;
echo "Prune repository with keep${KEEP_INFO:1}";
${BORG_PRUNE};
else
${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" |
while read db; do
echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
# exclude checks
include=0;
if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
while read incl_db; do
if [ "${db}" = "${incl_db}" ]; then
include=1;
break;
fi;
done<"${BASE_FOLDER}${INCLUDE_FILE}";
else
include=1;
fi;
exclude=0;
if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
while read excl_db; do
if [ "${db}" = "${excl_db}" ]; then
exclude=1;
break;
fi;
done<"${BASE_FOLDER}${EXCLUDE_FILE}";
fi;
if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then
# lock check
nolock='';
for nolock_db in $NOLOCKDB;
do
if [ "$nolock_db" = "$db" ];
then
nolock=$NOLOCKS;
fi;
done;
# event check
event='';
for event_db in $EVENTDB;
do
if [ "$event_db" = "$db" ];
then
event=$EVENTS;
fi;
done;
# set dump type
SCHEMA_ONLY=''; # empty for all
schema_flag='data'; # or data
if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then
while read schema_db; do
if [ "${db}" = "${schema_db}" ]; then
SCHEMA_ONLY='--no-data';
schema_flag='schema';
# skip out
break;
fi;
done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}";
fi;
# prepare borg calls
FILENAME="${db}-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql"
# backup set:
BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}";
BACKUP_SET_PREFIX="${db}-"
# borg call
BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
# debug or dry run
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} $nolock $event --opt ${SCHEMA_ONLY} --add-drop-database --databases ${db} | ${BORG_CALL};"
fi;
# backup
if [ ${DRYRUN} -eq 0 ]; then
$MYSQL_DUMP ${MYSQL_DB_CONFIG_PARAM} $nolock $event --opt ${SCHEMA_ONLY} --add-drop-database --databases ${db} | ${BORG_CALL};
_backup_error=$?;
if [ $_backup_error -ne 0 ]; then
echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
exit $_backup_error;
fi;
fi;
echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
${BORG_PRUNE};
else
echo "- [E] ${db}";
fi;
done;
fi;
. "${DIR}/borg.backup.functions.close.sh";
# __END__

View File

@@ -0,0 +1,11 @@
# Borg backup wrapper scripts settings
# override settings in borg.backup.settings with SUB_ prefix
# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_*
# set to 1 to dump all into one file instead of per database
# note that with this databases that have been dropped need to be pruned manually
# if 'schema' word is used, only schema data is dumped
DATABASE_FULL_DUMP="";
# override default postgres user
DATABASE_USER="";

229
borg.backup.pgsql.sh Executable file
View File

@@ -0,0 +1,229 @@
#!/usr/bin/env bash
# Backup PostgreSQL
# default is per table dump, can be set to one full dump
# config override set in borg.backup.pgsql.settings
# if run as postgres user, be sure user is in the backup group
# Run -I first to initialize repository
# There are no automatic repository checks unless -C is given
# set last edit date + time
MODULE="pgsql"
MODULE_VERSION="0.1.0";
DIR="${BASH_SOURCE%/*}"
if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
# init system
. "${DIR}/borg.backup.functions.init.sh";
# include and exclude file
INCLUDE_FILE="borg.backup.pgsql.include";
EXCLUDE_FILE="borg.backup.pgsql.exclude";
SCHEMA_ONLY_FILE="borg.backup.pgsql.schema-only";
BACKUP_INIT_CHECK="borg.backup.pgsql.init";
# check valid data
. "${DIR}/borg.backup.functions.check.sh";
# if info print info and then abort run
. "${DIR}/borg.backup.functions.info.sh";
if [ ! -z "${DATABASE_USER}" ]; then
DB_USER=${DATABASE_USER};
else
DB_USER='postgres';
fi;
# get current pgsql version first
# if first part <10 then user full, else only first part
# eg 9.4 -> 9.4, 12.5 -> 12
PG_VERSION=$(pgv=$(psql -U ${DB_USER} -d template1 -t -A -F "," -X -q -c 'select version();' | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/'); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi );
_PATH_PG_VERSION=${PG_VERSION};
_backup_error=$?;
if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then
echo "[! $(date +'%F %T')] Cannot get PostgreSQL server version: ${_backup_error}";
exit $_backup_error;
fi;
# path set per Distribution type and current running DB version
# Redhat: PG_BASE_PATH='/usr/pgsql-';
# AWS 1: PG_BASE_PATH='/usr/lib64/pgsql';
# Debian: PG_BASE_PATH='/usr/lib/postgresql/';
PG_BASE_PATH='/usr/lib/postgresql/';
if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
PG_BASE_PATH='/usr/pgsql-';
if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
PG_BASE_PATH='/usr/lib64/pgsql';
_PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//'):
if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
echo "[! $(date +'%F %T')] PostgreSQL not found in any paths";
exit 1;
fi;
fi;
fi;
PG_PATH=${PG_BASE_PATH}${_PATH_PG_VERSION}'/bin/';
PG_PSQL=${PG_PATH}'psql';
PG_DUMP=${PG_PATH}'pg_dump';
PG_DUMPALL=${PG_PATH}'pg_dumpall';
# check that command are here
if [ ! -f "${PG_PSQL}" ]; then
echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}";
exit 1;
fi;
if [ ! -f "${PG_DUMP}" ]; then
echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}";
exit 1;
fi;
if [ ! -f "${PG_DUMPALL}" ]; then
echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}";
exit 1;
fi;
DB_VERSION=${PG_VERSION};
# TODO override port/host info
DB_PORT='5432';
DB_HOST='local'; # or <host>
CONN_DB_HOST=''; # -h <host>
CONN_DB_PORT=''; # -p <port>
# borg call, replace ##...## parts
_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
# ALL IN ONE FILE or PER DATABASE FLAG
if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
SCHEMA_ONLY='';
schema_flag='data';
if [ "${DATABASE_FULL_DUMP}" = "schema" ]; then
SCHEMA_ONLY='-s';
schema_flag='schema';
fi;
echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
# Filename
FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
# backup set:
BACKUP_SET_NAME="all-${schema_flag}-${BACKUP_SET}";
BACKUP_SET_PREFIX="all-";
# borg call
BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}";
echo "${BORG_PRUNE}";
fi;
if [ ${DRYRUN} -eq 0 ]; then
$(${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL});
_backup_error=$?;
if [ $_backup_error -ne 0 ]; then
echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
exit $_backup_error;
fi;
fi;
echo "Prune repository with keep${KEEP_INFO:1}";
${BORG_PRUNE};
else
# dump globals first
db="pg_globals";
schema_flag="data";
echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
# Filename
FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
# backup set:
BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}";
BACKUP_SET_PREFIX="${db}-";
# borg call
BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}";
echo "${BORG_PRUNE}";
fi;
if [ ${DRYRUN} -eq 0 ]; then
${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL};
_backup_error=$?;
if [ $_backup_error -ne 0 ]; then
echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
exit $_backup_error;
fi;
fi;
echo "Prune repository with keep${KEEP_INFO:1}";
${BORG_PRUNE};
# get list of tables
for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do
# get the user who owns the DB too
owner=$(echo ${owner_db} | cut -d "," -f 1);
db=$(echo ${owner_db} | cut -d "," -f 2);
encoding=$(echo ${owner_db} | cut -d "," -f 3);
echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
include=0;
if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
while read incl_db; do
if [ "${db}" = "${incl_db}" ]; then
include=1;
break;
fi;
done<"${BASE_FOLDER}${INCLUDE_FILE}";
else
include=1;
fi;
exclude=0;
if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
while read excl_db; do
if [ "${db}" = "${excl_db}" ]; then
exclude=1;
break;
fi;
done<"${BASE_FOLDER}${EXCLUDE_FILE}";
fi;
if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then
# set dump type
SCHEMA_ONLY=''; # empty for all
schema_flag='data'; # or data
if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then
while read schema_db; do
if [ "${db}" = "${schema_db}" ]; then
SCHEMA_ONLY='-s';
schema_flag='schema';
# skip out
break;
fi;
done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}";
fi;
# Filename
# Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql
FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
# backup set:
BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}";
# PER db either data or schema
BACKUP_SET_PREFIX="${db}-";
# borg call
BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
# borg prune
BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}";
echo "${BORG_PRUNE}";
fi;
if [ ${DRYRUN} -eq 0 ]; then
${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL};
_backup_error=$?;
if [ $_backup_error -ne 0 ]; then
echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
exit $_backup_error;
fi;
fi;
echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
${BORG_PRUNE};
else
echo "- [E] ${db}";
fi;
done;
fi;
. "${DIR}/borg.backup.functions.close.sh";
# __END__

View File

@@ -0,0 +1,40 @@
# Borg backup wrapper scripts settings
# NOTE: ALL FOLDERS MUST BE ABSOLUTE WITH NO ~/ AT THE BEGINNING
# set log folder, if empty default will be used (/var/log/borg.backup)
LOG_FOLDER="";
# SSH user, host, port
# if ssh config is set only host needs to be filled
TARGET_USER="";
TARGET_HOST="";
TARGET_PORT="";
# if borg is not in default path, for ssh backup only
TARGET_BORG_PATH="";
# folder where the backup folder will be created
TARGET_FOLDER="";
# the backup file (folder) for this host $(hostname), must end with .borg
BACKUP_FILE="";
# compression settings (empty for none, lz4, zstd, zlib, lzma)
# level, if empty then default, else number between 0 and 9, or 1 to 22 for zstd
# default is zstd, 3
COMPRESSION="";
COMPRESSION_LEVEL="";
# encryption settings:
# SHA-256: 'none', 'authenticated', 'repokey', 'keyfile'
# BLAKE2b: 'authenticated-blake2', 'repokey-blake2', 'keyfile-blake2'
# Note: none or empty does not encrypt
# Blank passwords allowed for only key (if used, use keyfile)
# See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically
ENCRYPTION="";
# force repository check, default is off, set to true for check
FORCE_CHECK="";
# default is %Y-%m-%d
# todays date, if more than one per day add -%H%M for hour/minute
# it can also be "{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f}"
BACKUP_SET="";
# prune times, how many are kept in each time frame
KEEP_DAYS=7;
KEEP_WEEKS=4;
KEEP_MONTHS=6;
KEEP_YEARS=1;

115
borg.mount.sh Executable file
View File

@@ -0,0 +1,115 @@
#!/usr/bin/env bash
set -e -u -o pipefail
# mount this servers borg backup to a folder
# if no command is given the target folder is /mnt/restore
# if this folder does not exist, script will exit with an error
# base folder
BASE_FOLDER="/usr/local/scripts/backup/";
# borg settings file
SETTINGS_FILE="borg.backup.settings";
# base mount path (default)
MOUNT_PATH="/mnt/restore/";
# backup path to borg storage
ATTIC_BACKUP_FILE='';
# if we are mount or unmount (default is mount)
UMOUNT=0;
function usage ()
{
cat <<- EOT
Usage: ${0##/*/} [-c <config folder>] [-m <mount path>] [-f <borg backup file>] [-u <mount path]
-c <config folder>: if this is not given, ${BASE_FOLDER} is used
-m <mount path>: where to mount the image
-u umount mounted image
-f <borg backup file>: override full path to backup file instead of using the settings info
EOT
}
# set options
while getopts ":c:m:uf:h" opt do
case "${opt}" in
c|config)
BASE_FOLDER=${OPTARG};
;;
m|mount)
MOUNT_PATH=${OPTARG};
;;
u|umount)
UMOUNT=1;
;;
f|file)
ATTIC_BACKUP_FILE=${OPTARG};
;;
h|help)
usage;
exit 0;
;;
:)
echo "Option -$OPTARG requires an argument."
;;
\?)
echo -e "\n Option does not exist: ${OPTARG}\n";
usage;
exit 1;
;;
esac;
done;
if [ ! -d "${MOUNT_PATH}" ]; then
echo "The mount path ${MOUNT_PATH} cannot be found";
exit 0;
fi;
# add trailing slahd for base folder
[[ "${BASE_FOLDER}" != */ ]] && BASE_FOLDER="${BASE_FOLDER}/";
if [ ${UMOUNT} -eq 0 ]; then
TARGET_SERVER='';
if [ -z "${ATTIC_BACKUP_FILE}" ]; then
if [ ! -f "${BASE_FOLDER}${SETTINGS_FILE}" ]; then
echo "Cannot find ${BASE_FOLDER}${SETTINGS_FILE}";
exit 0;
fi;
. ${BASE_FOLDER}${SETTINGS_FILE}
# set the borg backup file base on the settings data
# if we have user/host then we build the ssh command
if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then
TARGET_SERVER=${TARGET_USER}"@"${TARGET_HOST}":";
fi;
REPOSITORY=${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE};
else
REPOSITORY=${ATTIC_BACKUP_FILE};
fi;
# check that the repostiory exists
REPOSITORY_OK=0;
if [ ! -z "${TARGET_SERVER}" ]; then
# remove trailing : for this
TARGET_SERVER=${TARGET_SERVER/:};
# use ssh command to check remote existense
if [ `ssh "${TARGET_SERVER}" "if [ -d \"${TARGET_FOLDER}${BACKUP_FILE}\" ]; then echo 1; else echo 0; fi;"` -eq 1 ]; then
REPOSITORY_OK=1;
fi;
elif [ -d "${REPOSITORY}" ]; then
REPOSITORY_OK=1;
fi;
if [ ${REPOSITORY_OK} -eq 0 ]; then
echo "Repository ${REPOSITORY} does not exists";
exit 0;
fi;
echo "Mounting ${REPOSITORY} on ${MOUNT_PATH}";
# all ok, lets mount it
borg mount "${REPOSITORY}" "${MOUNT_PATH}";
else
echo "Unmounting ${MOUNT_PATH}";
# will fail with error if not mounted, but not critical
borg umount "${MOUNT_PATH}";
fi;
## END