Borg Backup wrapper Version 4.0 update

* file backup borg folder has now -file name inside. Old folder must be
  manuall renamed
* All modules have the module id name as prefix in the backup set,
  _borg_backup_set_prefix_cleanup.sh needs to be run before to clean up
  all names or prune will not correctly delete old entries

New -T for one time target backup with custom prefix to have backups
outside the automated prune. -D option to delete this set

Add borg 1.2 support for compact which is called after delete and prune
to actually clean up the space.

-b borg executable and BORG_EXECUTEABLE override setting if borg is not
in path or another borg executable should be used

Readme.md

@@ -4,6 +4,29 @@ These scripts are wrappers around the main borg backup scripts.
 
 Modules for plain file backup, mysql and postgresql backup exists.
 
+## IMPORTANT NOTICE FOR UPGRADE TO VERSION 4.0 OR HIGHER
+
+*VERSION 4.0* CHANGE
+
+Version 4.0 introduces default borg repository name with `-file` for the `file` module. The repository has to be renamed manual before the next backup or the backup will fail.
+
+*Example:*
+
+Old backup name
+```sh
+BACKUP_FILE="some-backup-data.borg"
+```
+Then the file need to be renamed the following way:
+`mv some-backup-data.borg some-backup-data-file.borg`
+
+Below changes have to be done after the `file` module backup has been renamed.
+
+With Version 4.0 all backup sets are prefixed with the module name and a comma. For exmaple the files backup will have backup set "file,YYYY-MM-DD" as base name.
+
+To make sure prune of archives will work the `_borg_backup_set_prefix_cleanup.sh` script has to be run once. It has the same config (-c), debug (-d) and dry run (-n) options like the main scripts. It is recommended to run with the dry-run script first and see that the list of chagnes matches the expectation.
+
+The zabbix module has the prefix changed from `zabbix-settings-` to `zabbix,settings-` to match the new archive set rules
+
 ## Recommended setup
 
 git clone this repostory into the target folder:
@@ -16,6 +39,53 @@ Now the core scripts can be updated with a simple
 
 No settings files will be overwritten
 
+## Possible command line options
+
+### `-c <config folder>`
+if this is not given, /usr/local/scripts/borg/ is used
+
+### `-L <log folder>`
+override config set and default log folder
+
+### `-T <tag>`
+create one time stand alone backup prefixed with tag name
+
+### `-D <tag backup set>`
+remove a tagged backup set, full name must be given
+
+### `-b <borg executable>`
+override the default borg executable found in path
+
+### `-P`
+print list of archives created
+
+### `-C`
+check if repository exists, if not abort
+
+### `-E`
+exit after check
+
+### `-I`
+init repository (must be run first)
+
+### `-i`
+print out only info
+
+### `-l`
+list files during backup
+
+### `-v`
+be verbose
+
+### `-d`
+debug output all commands
+
+### `-n`
+only do dry run
+
+### `-h`
+this help page
+
 ## Basic Settings
 
 `borg.backup.settings`
@@ -60,7 +130,7 @@ All module settings files can have the following prefixed with `SUB_` to overrid
  * SUB_KEEP_YEARS
  * SUB_KEEP_WITHIN
 
-## Setup backup via SSH to remote host
+## Setup backup via SSH to remote host on `borg.backup.settings`
 
 For this the following settings are from interest
 
@@ -74,8 +144,16 @@ Note that if `.ssh/config` is used only `TARGET_HOST` needs to be set. Recommene
 
 and `TARGET_BORG_PATH="";` if the target borg is in a non default path
 
+## Override borg executable in `borg.backup.settings`
+
+`BORG_EXECUTABLE="<full path to borg>"`
+
 ## File backup settings
 
+On new setups it is recommended to use the `borg.backup.file.setings` and set
+`FILE_REPOSITORY_COMPATIBLE`
+to `true`
+
 ### Config variables
 
 

_borg_backup_set_prefix_cleanup.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+# this will fix backup sets name
+# must have a target call for
+# file
+# gitea
+# mysql
+# pgsql
+# zabbix-settings-
+
+export BORG_BASE_DIR="borg/";
+DRYRUN=0;
+TARGET_USER="";
+TARGET_HOST="";
+TARGET_PORT="";
+TARGET_BORG_PATH="";
+TARGET_FOLDER="";
+BASE_FOLDER="/usr/local/scripts/borg/";
+# those are the valid modules
+MODULE_LIST="file gitea mysql pgsql zabbix"
+
+# basic options
+# -c for config file override
+# -n for dry run test
+while getopts ":c:nd" opt; do
+	case "${opt}" in
+		c|config)
+			BASE_FOLDER=${OPTARG};
+			;;
+		d|debug)
+			DEBUG=1;
+			;;
+		n|dryrun)
+			DRYRUN=1;
+			;;
+		:)
+			echo "Option -$OPTARG requires an argument."
+			;;
+		\?)
+			echo -e "\n Option does not exist: ${OPTARG}\n";
+			usage;
+			exit 1;
+			;;
+	esac;
+done;
+
+
+if [ ! -d "${BASE_FOLDER}" ]; then
+	echo "Base folder not found: ${BASE_FOLDER}";
+	exit 1;
+fi;
+SETTINGS_FILE="borg.backup.settings";
+
+. "${BASE_FOLDER}${SETTINGS_FILE}";
+ORIG_BACKUPFILE=${BACKUP_FILE};
+for MODULE in ${MODULE_LIST}; do
+	echo "************* MODULE: ${MODULE}";
+	BACKUP_FILE=${ORIG_BACKUPFILE};
+	# if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then
+	# 	. "${BASE_FOLDER}${SETTINGS_FILE_SUB}";
+	# fi;
+	# SETTINGS_FILE_SUB=$(echo "${SETTINGS_FILE}" | sed -e "s/\.settings/\.${MODULE,,}\.settings/");
+	if [ "${MODULE,,}" != "file" ]; then
+		BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg};
+	fi;
+	TARGET_FOLDER=${TARGET_FOLDER%/}
+	TARGET_FOLDER=${TARGET_FOLDER#/}
+	# and add slash front and back and escape the path
+	TARGET_FOLDER=$(printf "%q" "/${TARGET_FOLDER}/");
+	if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then
+		TARGET_SERVER="ssh://${TARGET_USER}@${TARGET_HOST}:${TARGET_PORT}/";
+	# host/port
+	elif [ ! -z "${TARGET_HOST}" ] && [ ! -z "${TARGET_PORT}" ]; then
+		TARGET_SERVER="ssh://${TARGET_HOST}:${TARGET_PORT}/";
+	# user/host
+	elif [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then
+		TARGET_SERVER="${TARGET_USER}@${TARGET_HOST}:";
+	# host
+	elif [ ! -z "${TARGET_HOST}" ]; then
+		TARGET_SERVER="${TARGET_HOST}:";
+	fi;
+	# we dont allow special characters, so we don't need to special escape it
+	REPOSITORY="${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE}";
+	echo "==== REPOSITORY: ${REPOSITORY}";
+	borg list --format '{archive}{NL}' ${REPOSITORY}|grep -v "${MODULE},"|
+	while read i; do
+		# for gitea, zabbix we do not ADD we RENAME
+		if [ "${MODILE}" = "gitea" ]; then
+			target_name=$(echo $i | sed -e 's/gitea-/gitea,/');
+		elif [ "${MODILE}" = "zabbix" ]; then
+			target_name=$(echo $i | sed -e 's/zabbix-settings-/zabbix,settings-/');
+		else
+			target_name="${MODULE},${i}";
+		fi;
+		echo "- Rename from: ${i} to: ${target_name}";
+		if [ ${DEBUG} -eq 1 ]; then
+			echo "borg rename -p -v ${REPOSITORY}::${i} ${target_name}";
+		fi;
+		if [ ${DRYRUN} -eq 0 ]; then
+			borg rename -p -v ${REPOSITORY}::${i} ${target_name};
+		fi;
+	done;
+done;
+
+# __END__

borg.backup.file.settings-default

@@ -2,8 +2,8 @@
 
 # rename to borg.backup.file.settings to use
 
-# set to false to use -file, current default is "true"
+# set to true to use old setting without -file in backup name
-#FILE_REPOSITORY_COMPATIBLE="false"
+#FILE_REPOSITORY_COMPATIBLE="true"
 
 # override settings in borg.backup.settings with SUB_ prefix
 # valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_*

borg.backup.file.sh

@@ -5,7 +5,7 @@
 
 # set last edit date + time
 MODULE="file";
-MODULE_VERSION="1.0.0";
+MODULE_VERSION="1.2.0";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -22,6 +22,7 @@ BACKUP_INIT_CHECK="borg.backup.file.init";
 # exit if include file is missing
 if [ ! -f "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
 	echo "[! $(date +'%F %T')] The include folder file ${INCLUDE_FILE} is missing";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 echo "--- [INCLUDE: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
@@ -146,11 +147,14 @@ if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
 		OPT_EXCLUDE="--exclude-from=${TMP_EXCLUDE_FILE}";
 	fi;
 fi;
+
+# set a special file prefix
+BACKUP_SET_PREFIX="${MODULE},";
 # add the repository set before we add the folders
 # base command
-COMMAND="borg create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} ";
+COMMAND="${BORG_COMMAND} create -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s ${OPT_REMOTE} ${OPT_EXCLUDE} ";
 # add repoistory, after that the folders will be added on call
-COMMAND=${COMMAND}${REPOSITORY}::${BACKUP_SET};
+COMMAND=${COMMAND}${REPOSITORY}::${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET};
 # if info print info and then abort run
 . "${DIR}/borg.backup.functions.info.sh";
 
@@ -172,19 +176,26 @@ if [ $FOLDER_OK -eq 1 ]; then
 	fi;
 else
 	echo "[! $(date +'%F %T')] No folders where set for the backup";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 
-# clean up, always verbose
+# clean up, always verbose, but only if we do not run one time tag
-echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+if [ -z "${ONE_TIME_TAG}" ]; then
-# build command
+	echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
-COMMAND="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} ${KEEP_OPTIONS[*]} ${REPOSITORY}";
+	# build command
-echo "Prune repository with keep${KEEP_INFO:1}";
+	COMMAND="${BORG_COMMAND} prune ${OPT_REMOTE} -v --list ${OPT_PROGRESS} ${DRY_RUN_STATS} -P ${BACKUP_SET_PREFIX} ${KEEP_OPTIONS[*]} ${REPOSITORY}";
-if [ ${DEBUG} -eq 1 ]; then
+	echo "Prune repository with keep${KEEP_INFO:1}";
-	echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g';
+	if [ ${DEBUG} -eq 1 ]; then
+		echo "${COMMAND//#/ }" | sed -e 's/[ ][ ]*/ /g';
+	fi;
+	# for the IFS="#" to work we need to replace options spaces with exactly ONE #
+	$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted";
+	# if this is borg version >1.2 we need to run compact after prune
+	. "${DIR}/borg.backup.functions.compact.sh";
+else
+	echo "[#] No prune with tagged backup";
 fi;
-# for the IFS="#" to work we need to replace options spaces with exactly ONE #
-$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted";
 
 . "${DIR}/borg.backup.functions.close.sh";
 

borg.backup.functions.check.sh

@@ -10,6 +10,8 @@ echo "Script version: ${VERSION}";
 # show type
 echo "Backup module : ${MODULE}";
 echo "Module version: ${MODULE_VERSION}";
+# borg version
+echo "Borg version  : ${BORG_VERSION}";
 # show base folder always
 echo "Base folder   : ${BASE_FOLDER}";
 
@@ -130,63 +132,71 @@ if [ ! -w "${HOME}" ] || [ "${HOME}" = '/' ]; then
 	HOME=$(eval echo "$(whoami)");
 fi;
 
-# build options and info string,
+# keep optionfs (for files)
-# also flag BACKUP_SET check if hourly is set
 KEEP_OPTIONS=();
+# keep info string (for files)
 KEEP_INFO="";
-BACKUP_SET_CHECK=0;
+# override standard keep for tagged backups
-if [ ${KEEP_LAST} -gt 0 ]; then
+if [ ! -z "${ONE_TIME_TAG}" ]; then
-	KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}");
+	BACKUP_SET="{now:%Y-%m-%dT%H:%M:%S}";
-	KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}";
+else
-fi;
+	# build options and info string,
-if [ ${KEEP_HOURS} -gt 0 ]; then
+	# also flag BACKUP_SET check if hourly is set
-	KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}");
+	BACKUP_SET_CHECK=0;
-	KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}";
+	if [ ${KEEP_LAST} -gt 0 ]; then
-	BACKUP_SET_CHECK=1;
+		KEEP_OPTIONS+=("--keep-last=${KEEP_LAST}");
-fi;
+		KEEP_INFO="${KEEP_INFO}, last: ${KEEP_LAST}";
-if [ ${KEEP_DAYS} -gt 0 ]; then
+	fi;
-	KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}");
+	if [ ${KEEP_HOURS} -gt 0 ]; then
-	KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}";
+		KEEP_OPTIONS+=("--keep-hourly=${KEEP_HOURS}");
-fi;
+		KEEP_INFO="${KEEP_INFO}, hourly: ${KEEP_HOURS}";
-if [ ${KEEP_WEEKS} -gt 0 ]; then
+		BACKUP_SET_CHECK=1;
-	KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}");
+	fi;
-	KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}";
+	if [ ${KEEP_DAYS} -gt 0 ]; then
-fi;
+		KEEP_OPTIONS+=("--keep-daily=${KEEP_DAYS}");
-if [ ${KEEP_MONTHS} -gt 0 ]; then
+		KEEP_INFO="${KEEP_INFO}, daily: ${KEEP_DAYS}";
-	KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}");
+	fi;
-	KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}";
+	if [ ${KEEP_WEEKS} -gt 0 ]; then
-fi;
+		KEEP_OPTIONS+=("--keep-weekly=${KEEP_WEEKS}");
-if [ ${KEEP_YEARS} -gt 0 ]; then
+		KEEP_INFO="${KEEP_INFO}, weekly: ${KEEP_WEEKS}";
-	KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}");
+	fi;
-	KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}";
+	if [ ${KEEP_MONTHS} -gt 0 ]; then
-fi;
+		KEEP_OPTIONS+=("--keep-monthly=${KEEP_MONTHS}");
-if [ ! -z "${KEEP_WITHIN}" ]; then
+		KEEP_INFO="${KEEP_INFO}, monthly: ${KEEP_MONTHS}";
-	# check for invalid string. can only be number + H|d|w|m|y
+	fi;
-	if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then
+	if [ ${KEEP_YEARS} -gt 0 ]; then
-		KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}");
+		KEEP_OPTIONS+=("--keep-yearly=${KEEP_YEARS}");
-		KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}";
+		KEEP_INFO="${KEEP_INFO}, yearly: ${KEEP_YEARS}";
-		if [[ "${KEEP_WITHIN}" == *"H"* ]]; then
+	fi;
-			BACKUP_SET_CHECK=1;
+	if [ ! -z "${KEEP_WITHIN}" ]; then
+		# check for invalid string. can only be number + H|d|w|m|y
+		if [[ "${KEEP_WITHIN}" =~ ^[0-9]+[Hdwmy]{1}$ ]]; then
+			KEEP_OPTIONS+=("--keep-within=${KEEP_WITHIN}");
+			KEEP_INFO="${KEEP_INFO}, within: ${KEEP_WITHIN}";
+			if [[ "${KEEP_WITHIN}" == *"H"* ]]; then
+				BACKUP_SET_CHECK=1;
+			fi;
+		else
+			echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string.";
+			exit 1;
 		fi;
-	else
+	fi;
-		echo "[! $(date +'%F %T')] KEEP_WITHIN has invalid string.";
+	# abort if KEEP_OPTIONS is empty
+	if [ -z "${KEEP_OPTIONS}" ]; then
+		echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format.";
 		exit 1;
 	fi;
+	# set BACKUP_SET if empty, set to Year-month-day
+	if [ -z "${BACKUP_SET}" ]; then
+		BACKUP_SET="{now:%Y-%m-%d}";
+	fi;
+	# backup set check, and there is no hour entry (%H) in the archive string
+	# we add T%H:%M:%S in this case, before the last }
+	if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then
+		BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/");
+	fi;
 fi;
-# abort if KEEP_OPTIONS is empty
+
-if [ -z "${KEEP_OPTIONS}" ]; then
-	echo "[! $(date +'%F %T')] It seems no KEEP_* entries where set in a valid format.";
-	exit 1;
-fi;
-# set BACKUP_SET if empty, set to Year-month-day
-if [ -z "${BACKUP_SET}" ]; then
-	BACKUP_SET="{now:%Y-%m-%d}";
-fi;
-# backup set check, and there is no hour entry (%H) in the archive string
-# we add T%H:%M:%S in this case, before the last }
-if [ ${BACKUP_SET_CHECK} -eq 1 ] && [[ "${BACKUP_SET}" != *"%H"* ]]; then
-	BACKUP_SET=$(echo "${BACKUP_SET}" | sed -e "s/}/T%H:%M:%S}/");
-fi;
 
 # for folders list split set to "#" and keep the old setting as is
 _IFS=${IFS};
@@ -198,8 +208,8 @@ fi;
 
 # borg call, replace ##...## parts during run
 # used in all modules, except 'file'
-_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
+_BORG_CALL="${BORG_COMMAND} create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
-_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
+_BORG_PRUNE="${BORG_COMMAND} prune ${OPT_REMOTE} -v --list ${OPT_PROGRESS} ${DRY_RUN_STATS} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
 
 # general borg settings
 # set base path to config directory to keep cache/config separated
@@ -216,7 +226,7 @@ if [ ${DEBUG} -eq 1 ]; then
 fi;
 # prepare debug commands only
 COMMAND_EXPORT="export BORG_BASE_DIR=\"${BASE_FOLDER}\";"
-COMMAND_INFO="${COMMAND_EXPORT}borg info ${OPT_REMOTE} ${REPOSITORY}";
+COMMAND_INFO="${COMMAND_EXPORT}${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}";
 # if the is not there, call init to create it
 # if this is user@host, we need to use ssh command to check if the file is there
 # else a normal check is ok
@@ -225,14 +235,13 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then
 	echo "--- [CHECK : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	if [ ! -z "${TARGET_SERVER}" ]; then
 		if [ ${DEBUG} -eq 1 ]; then
-			echo "borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\"";
+			echo "${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\"";
 		fi;
 		# use borg info and check if it returns "Repository ID:" in the first line
-		REPO_CHECK=$(borg info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:");
+		REPO_CHECK=$(${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep "Repository ID:");
 		# this is currently a hack to work round the error code in borg info
 		# this checks if REPO_CHECK holds this error message and then starts init
-		regex="^Some part of the script failed with an error:";
+		if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${REGEX_ERROR} ]]; then
-		if [[ -z "${REPO_CHECK}" ]] || [[ "${REPO_CHECK}" =~ ${regex} ]]; then
 			INIT_REPOSITORY=1;
 		fi;
 	elif [ ! -d "${REPOSITORY}" ]; then
@@ -248,37 +257,39 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ]; then
 	# end if checked but repository is not here
 	if [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then
 		echo "[! $(date +'%F %T')] No repository. Please run with -I flag to initialze repository";
+		. "${DIR}/borg.backup.functions.close.sh" 1;
 		exit 1;
 	fi;
 	if [ ${EXIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INIT} -eq 0 ]; then
 		echo "Repository exists";
 		echo "For more information run:"
 		echo "${COMMAND_INFO}";
-		echo "=== [END  : $(date +'%F %T')] ==[${MODULE}]====================================>";
+		. "${DIR}/borg.backup.functions.close.sh";
 		exit;
 	fi;
 fi;
 if [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 1 ]; then
 	echo "--- [INIT  : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
-		echo "borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}";
+		echo "${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY}";
 	fi
 	if [ ${DRYRUN} -eq 0 ]; then
 		# should trap and exit properly here
-		borg init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY};
+		${BORG_COMMAND} init ${OPT_REMOTE} -e ${ENCRYPTION} ${OPT_VERBOSE} ${REPOSITORY};
 		# write init file
 		echo "$(date +%s)" > "${BASE_FOLDER}${BACKUP_INIT_CHECK}";
 		echo "Repository initialized";
 		echo "For more information run:"
 		echo "${COMMAND_INFO}";
 	fi
-	echo "=== [END  : $(date +'%F %T')] ==[${MODULE}]====================================>";
+	. "${DIR}/borg.backup.functions.close.sh";
 	# exit after init
 	exit;
 elif [ ${INIT} -eq 1 ] && [ ${INIT_REPOSITORY} -eq 0 ]; then
 	echo "[! $(date +'%F %T')] Repository already initialized";
 	echo "For more information run:"
 	echo "${COMMAND_INFO}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 
@@ -286,6 +297,7 @@ fi;
 if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_CHECK}" ]; then
 	echo "[! $(date +'%F %T')] It seems the repository has never been initialized."
 	echo "Please run -I to initialize or if already initialzed run with -C for init update."
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 
@@ -295,7 +307,7 @@ if [ ${PRINT} -eq 1 ]; then
 	FORMAT="{archive} {comment:6} {start} - {end} [{id}] ({username}@{hostname}){NL}"
 	# show command on debug or dry run
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} list ${OPT_REMOTE} --format ${FORMAT} ${REPOSITORY}";
 	fi;
 	# run info command if not a dry drun
 	if [ ${DRYRUN} -eq 0 ]; then
@@ -304,17 +316,60 @@ if [ ${PRINT} -eq 1 ]; then
 	if [ ${VERBOSE} -eq 1 ]; then
 		echo "";
 		echo "Base command info:"
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} [COMMAND] ${OPT_REMOTE} ${REPOSITORY}::[BACKUP] [PATH]";
 		echo "Replace [COMMAND] with list for listing or extract for restoring backup data."
 		echo "Replace [BACKUP] with archive name."
 		echo "If no [PATH] is given then all files will be restored."
 		echo "Before extracting -n (dry run) is recommended to use."
 		echo "If archive size is needed the info command with archive name has to be used."
-		echo "When listing (list) data the --format command can be used."
+		echo "When listing files in an archive set (::SET) the --format command can be used."
 		echo "Example: \"{mode} {user:6} {group:6} {size:8d} {csize:8d} {dsize:8d} {dcsize:8d} {mtime} {path}{extra} [{health}]{NL}\""
 	else
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} [COMMAND] ${OPT_REMOTE} [FORMAT] ${REPOSITORY}::[BACKUP] [PATH]";
 	fi;
+	. "${DIR}/borg.backup.functions.close.sh";
+	exit;
+fi;
+
+# DELETE ONE TIME TAG
+if [ ! -z "${DELETE_ONE_TIME_TAG}" ]; then
+	echo "--- [DELETE: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	# if a "*" is inside we don't do ONE archive, but globbing via -a option
+	DELETE_ARCHIVE=""
+	OPT_GLOB="";
+	# this is more or less for debug only
+	if [[ "${DELETE_ONE_TIME_TAG}" =~ $REGEX_GLOB ]]; then
+		OPT_GLOB="-a '${DELETE_ONE_TIME_TAG}'"
+	else
+		DELETE_ARCHIVE="::"${DELETE_ONE_TIME_TAG};
+	fi
+	# if this is borg <1.2 OPT_LIST does not work
+	if [ $(version $BORG_VERSION) -lt $(version "1.2.0") ]; then
+		OPT_LIST="";
+	fi;
+	# if exists, delete and exit
+	# show command on debug or dry run
+	if [ ${DEBUG} -eq 1 ]; then
+		echo "${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} -s ${OPT_GLOB} ${REPOSITORY}${DELETE_ARCHIVE}";
+	fi;
+	# run delete command if not a dry drun
+	# NOTE seems to be glob is not working if wrapped into another variable
+	if [[ "${DELETE_ONE_TIME_TAG}" =~ $REGEX_GLOB ]]; then
+		${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} ${DRY_RUN_STATS} -a "${DELETE_ONE_TIME_TAG}" ${REPOSITORY};
+	else
+		${BORG_COMMAND} delete ${OPT_REMOTE} ${OPT_LIST} ${DRY_RUN_STATS} ${REPOSITORY}${DELETE_ARCHIVE};
+	fi;
+	# if not a dry run, compact repository after delete
+	# not that compact only works on borg 1.2
+	if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
+		if [ ${DRYRUN} -eq 0 ]; then
+			${BORG_COMMAND} compact ${REPOSITORY};
+		fi;
+		if [ ${DEBUG} -eq 1 ]; then
+			echo "${BORG_COMMAND} compact ${REPOSITORY}";
+		fi;
+	fi;
+	. "${DIR}/borg.backup.functions.close.sh";
 	exit;
 fi;
 

borg.backup.functions.close.sh

@@ -2,8 +2,13 @@
 
 # unset borg settings
 unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK
-DURATION=$[ $(date +'%s')-$START ];
+# error abort without duration and error notice
-echo "=== [Run time: $(convert_time ${DURATION})]";
+if [ $1 == 1 ]; then
-echo "=== [END  : $(date +'%F %T')] ==[${MODULE}]===================>";
+	echo "=== [ERROR: $(date +'%F %T')] ==[${MODULE}]====================================>";
+else
+	DURATION=$[ $(date +'%s')-$START ];
+	echo "=== [Run time: $(convert_time ${DURATION})]";
+	echo "=== [END  : $(date +'%F %T')] ==[${MODULE}]====================================>";
+fi;
 
 # __END__

borg.backup.functions.compact.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# compact (only if BORG COMPACT is set)
+# only for borg 1.2
+
+# reset to normal IFS, so command works here
+IFS=${_IFS};
+if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
+	echo "--- [COMPACT:$(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}";
+	if [ ${DEBUG} -eq 1 ]; then
+			echo "${BORG_COMPACT}";
+	fi;
+	if [ ${DRYRUN} -eq 0 ]; then
+		${BORG_COMPACT};
+	fi;
+fi;
+
+# __END__

borg.backup.functions.info.sh

@@ -4,11 +4,11 @@ if [ ${INFO} -eq 1 ]; then
 	echo "--- [INFO  : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	# show command on debug or dry run
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";borg info ${OPT_REMOTE} ${REPOSITORY}";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}";
 	fi;
 	# run info command if not a dry drun
 	if [ ${DRYRUN} -eq 0 ]; then
-		borg info ${OPT_REMOTE} ${REPOSITORY};
+		${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY};
 	fi;
 	if [ "${MODULE}" = "files" ]; then
 		if [ $FOLDER_OK -eq 1 ]; then
@@ -23,7 +23,7 @@ if [ ${INFO} -eq 1 ]; then
 			rm -f "${TMP_EXCLUDE_FILE}";
 		fi;
 	fi;
-	echo "=== [END  : $(date +'%F %T')] ==[${MODULE}]====================================>";
+	. "${DIR}/borg.backup.functions.close.sh";
 	exit;
 fi;
 

borg.backup.functions.init.sh

@@ -13,10 +13,17 @@ cleanup() {
 }
 # on exit unset any exported var
 trap "unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK" EXIT;
+# for version compare
+function version {
+	echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
+}
 
 # version for all general files
-VERSION="3.0.2";
+VERSION="4.1.0";
 
+# borg version and borg comamnd
+BORG_VERSION="";
+BORG_COMMAND="borg";
 # default log folder if none are set in config or option
 _LOG_FOLDER="/var/log/borg.backup/";
 # log file name is set based on BACKUP_FILE, .log is added
@@ -34,6 +41,9 @@ INCLUDE_FILE="";
 EXCLUDE_FILE="";
 # backup folder initialzed check
 BACKUP_INIT_CHECK="";
+# one time backup prefix tag, if set will use <tag>.<prefix>-Y-M-DTh:m:s type backup prefix
+ONE_TIME_TAG="";
+DELETE_ONE_TIME_TAG="";
 # debug/verbose
 VERBOSE=0;
 LIST=0;
@@ -51,13 +61,14 @@ _BORG_RELOCATED_REPO_ACCESS_IS_OK="yes";
 # NOTE: to keep the old .borg repository name for file module set this to true
 # if set to false (future) it will add -file to the repository name like for other
 # modules
-FILE_REPOSITORY_COMPATIBLE="true";
+FILE_REPOSITORY_COMPATIBLE="false";
 # other variables
 TARGET_SERVER="";
 REGEX="";
 REGEX_COMMENT="^[\ \t]*#";
 REGEX_GLOB='\*';
 REGEX_NUMERIC="^[0-9]{1,2}$";
+REGEX_ERROR="^Some part of the script failed with an error:";
 PRUNE_DEBUG="";
 INIT_REPOSITORY=0;
 FOLDER_OK=0;
@@ -77,6 +88,10 @@ TARGET_BORG_PATH="";
 TARGET_FOLDER="";
 BACKUP_FILE="";
 SUB_BACKUP_FILE="";
+# OPT is for options set
+OPT_BORG_EXECUTEABLE="";
+# which overrides BORG_EXECUTABLE that can be set in the settings file
+BORG_EXECUTEABLE="";
 # lz4, zstd 1-22 (3), zlib 0-9 (6), lzma 0-9 (6)
 DEFAULT_COMPRESSION="zstd";
 DEFAULT_COMPRESSION_LEVEL=3;
@@ -147,13 +162,16 @@ function usage()
 
 	-c <config folder>: if this is not given, ${BASE_FOLDER} is used
 	-L <log folder>: override config set and default log folder
+	-T <tag>: create one time stand alone backup prefixed with tag name
+	-D <tag backup set>: remove a tagged backup set, full name must be given
+	-b <borg executable>: override default path
 	-P: print list of archives created
 	-C: check if repository exists, if not abort
 	-E: exit after check
 	-I: init repository (must be run first)
-	-v: be verbose
 	-i: print out only info
 	-l: list files during backup
+	-v: be verbose
 	-d: debug output all commands
 	-n: only do dry run
 	-h: this help page
@@ -165,14 +183,23 @@ function usage()
 }
 
 # set options
-while getopts ":c:L:vldniCEIPh" opt; do
+while getopts ":c:L:T:D:b:vldniCEIPh" opt; do
 	case "${opt}" in
 		c|config)
 			BASE_FOLDER=${OPTARG};
 			;;
-		L|log)
+		L|Log)
 			OPT_LOG_FOLDER=${OPTARG};
 			;;
+		T|Tag)
+			ONE_TIME_TAG=${OPTARG};
+			;;
+		D|Delete)
+			DELETE_ONE_TIME_TAG=${OPTARG};
+			;;
+		b|borg)
+			OPT_BORG_EXECUTEABLE=${OPTARG};
+			;;
 		C|Check)
 			# will check if repo is there and abort if not
 			CHECK=1;
@@ -244,10 +271,36 @@ if [ ${CHECK} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${INFO} -eq 1 ]; then
 	exit 1;
 fi;
 # print -P cannot be run with -i/-C/-I together
-if [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] && [ ${CHECK} -eq 1 ] && [ ${INFO} -eq 1 ]; then
+if [ ${PRINT} -eq 1 ] && ([ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then
 	echo "Cannot have -P print option and -i info, -C check or -I initizalize option at the same time";
 	exit 1;
 fi;
+# if tag is set, you can't have init, check, info, etc
+if [ ! -z "${ONE_TIME_TAG}" ] && ([ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then
+	echo "Cannot have -T '${ONE_TIME_TAG}' option with -i info, -C check, -I initialize or -P print option at the same time";
+	exit 1;
+fi;
+# check only alphanumeric, no spaces, only underscore and dash
+if [ ! -z "${ONE_TIME_TAG}" ] && ! [[ "${ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+$ ]]; then
+	echo "One time tag '${ONE_TIME_TAG}' must be alphanumeric with dashes and underscore only.";
+	exit 1;
+elif [ ! -z "${ONE_TIME_TAG}" ]; then
+	# all ok, attach . at the end
+	ONE_TIME_TAG=${ONE_TIME_TAG}".";
+fi;
+# if -D, cannot be with -T, -i, -C, -I, -P
+if [ ! -z "${DELETE_ONE_TIME_TAG}" ] && ([ ! -z "${ONE_TIME_TAG}" ] || [ ${PRINT} -eq 1 ] || [ ${INIT} -eq 1 ] || [ ${CHECK} -eq 1 ] || [ ${INFO} -eq 1 ]); then
+	echo "Cannot have -D delete tag option with -T one time tag, -i info, -C check, -I initialize or -P print option at the same time";
+	exit 1;
+fi;
+# -D also must be in valid backup set format
+# ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]]
+if [ ! -z "${DELETE_ONE_TIME_TAG}" ] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},([A-Za-z0-9_-]+-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$ ]] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]]; then
+	echo "Delete one time tag '${DELETE_ONE_TIME_TAG}' is in an invalid format. Please check existing tags with -P option."
+	echo "For a globing be sure it is in the format of: TAG.MODULE,*-YYYY-MM-DDT*";
+	echo "Note the dash (-) after the first *, also time (T) is a globa (*) must."
+	exit 1;
+fi;
 
 # verbose & progress
 if [ ${VERBOSE} -eq 1 ]; then
@@ -258,12 +311,45 @@ fi;
 if [ ${LIST} -eq 1 ]; then
 	OPT_LIST="--list";
 fi;
+# If dry run, the stats (-s) option cannot be used
 if [ ${DRYRUN} -eq 1 ]; then
-	PRUNE_DEBUG="--dry-run";
+	DRY_RUN_STATS="-n";
+else
+	DRY_RUN_STATS="-s";
 fi;
 
 # read config file
 . "${BASE_FOLDER}${SETTINGS_FILE}";
+
+# if OPTION SET overrides ALL others
+if [ ! -z "${OPT_BORG_EXECUTEABLE}" ]; then
+	BORG_COMMAND="${OPT_BORG_EXECUTEABLE}";
+	if [ ! -f "${BORG_COMMAND}" ]; then
+		echo "borg command not found with option -b: ${BORG_COMMAND}";
+		exit;
+	fi;
+# if in setting file, use this
+elif [ ! -z "${BORG_EXECUTEABLE}" ]; then
+	BORG_COMMAND="${BORG_EXECUTEABLE}";
+	if [ ! -f "${BORG_COMMAND}" ]; then
+		echo "borg command not found with setting: ${BORG_COMMAND}";
+		exit;
+	fi;
+elif ! command -v borg &> /dev/null; then
+	echo "borg backup seems not to be installed, please check paths";
+	exit;
+fi;
+# check that this is a borg executable, no detail check
+_BORG_COMMAND_CHECK=$(${BORG_COMMAND} -V | grep "borg");
+if [[ "${_BORG_COMMAND_CHECK}" =~ ${REGEX_ERROR} ]]; then
+	echo "Cannot extract borg info from command, is this a valid borg executable?: ${BORG_COMMAND}";
+	exit;
+fi;
+# extract actually borg version from here
+# alt sed to get only numbes: sed -e 's/.* \([0-9]*\.[0-9]*\.[0-9]*\)$/\1/g'
+# or use cut -d " " -f 2 and assume NO space in the first part
+BORG_VERSION=$(${BORG_COMMAND} -V | sed -e 's/borg.* //') 2>&1 || echo "[!] Borg version not estable";
+
 # load default settings for fileds not set
 if [ -z "${COMPRESSION}" ]; then
 	COMPRESSION="${DEFAULT_COMPRESSION}";
@@ -341,7 +427,7 @@ if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then
 fi;
 # add module name to backup file, always
 # except if FILE module and FILE_REPOSITORY_COMPATIBLE="true"
-if ([ "${FILE_REPOSITORY_COMPATIBLE}" = "false" ] && [ "${MODULE,,}" = "file" ]) || [ "${MODULE,,}" != "file" ]; then
+if [ "${FILE_REPOSITORY_COMPATIBLE}" != "true" ] || [ "${MODULE,,}" != "file" ]; then
 	BACKUP_FILE=${BACKUP_FILE/.borg/-${MODULE,,}.borg};
 fi;
 # backup file must be set

borg.backup.gitea.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 MODULE="gitea"
-MODULE_VERSION="1.0.0";
+MODULE_VERSION="1.1.1";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -35,17 +35,19 @@ if [ -z "${GITEA_CONFIG}" ]; then
 fi;
 if [ ! -f "${GITEA_BIN}" ]; then
 	echo "[! $(date +'%F %T')] Cannot find gitea binary";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 if [ ! -f "${GITEA_CONFIG}" ]; then
 	echo "[! $(date +'%F %T')] Cannot find gitea config";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 # Filename
 FILENAME="gitea.backup.zip";
 # backup set and prefix
-BACKUP_SET_PREFIX="gitea-";
+BACKUP_SET_PREFIX="${MODULE},";
-BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${BACKUP_SET}";
+BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}";
 
 # borg call
 BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
@@ -53,7 +55,9 @@ BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SE
 echo "--- [git data and database: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 	echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}";
-	echo "${BORG_PRUNE}";
+	if [ -z "${ONE_TIME_TAG}" ]; then
+		echo "${BORG_PRUNE}";
+	fi;
 fi;
 if [ ${DRYRUN} -eq 0 ]; then
 	(
@@ -69,8 +73,13 @@ if [ ${DRYRUN} -eq 0 ]; then
 		sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL};
 	) | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' # remove all ESC strings
 fi;
-echo "Prune repository with keep${KEEP_INFO:1}";
+if [ -z "${ONE_TIME_TAG}" ]; then
-${BORG_PRUNE};
+	echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	echo "Prune repository with keep${KEEP_INFO:1}";
+	${BORG_PRUNE};
+	# if this is borg version >1.2 we need to run compact after prune
+	. "${DIR}/borg.backup.functions.compact.sh";
+fi;
 
 . "${DIR}/borg.backup.functions.close.sh";
 

borg.backup.mysql.sh

@@ -10,7 +10,7 @@
 
 # set last edit date + time
 MODULE="mysql"
-MODULE_VERSION="1.0.0";
+MODULE_VERSION="1.1.0";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -42,10 +42,12 @@ MYSQL_CMD=${MYSQL_BASE_PATH}'mysql';
 # no dump or mysql, bail
 if [ ! -f "${MYSQL_DUMP}" ]; then
 	echo "[! $(date +'%F %T')] mysqldump binary not found";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 if [ ! -f "${MYSQL_CMD}" ]; then
 	echo "[! $(date +'%F %T')] mysql binary not found";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 # check that the user can actually do, else abort here
@@ -54,6 +56,7 @@ _MYSQL_CHECK=$(mysqladmin ${MYSQL_DB_CONFIG_PARAM} ping 2>&1);
 _MYSQL_OK=$(echo "${_MYSQL_CHECK}" | grep "is alive");
 if [ -z "${_MYSQL_OK}" ]; then
 	echo "[! $(date +'%F %T')] Current user has no access right to mysql database";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 # below is for file name only
@@ -89,35 +92,42 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		SCHEMA_ONLY='--no-data';
 		schema_flag='schema';
 	fi;
-	echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	echo "---[BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	# We only do a full backup and not per table backup here
 	# Filename
 	FILENAME="all-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql"
 	# backup set:
-	BACKUP_SET_PREFIX="all-";
+	BACKUP_SET_PREFIX="${MODULE},all-";
-	BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
+	BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 	# borg call
 	BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
 	BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
 		echo "${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL}";
-		echo "${BORG_PRUNE}";
+		if [ -z "${ONE_TIME_TAG}" ]; then
+			echo "${BORG_PRUNE}";
+		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
 		${MYSQL_DUMP} ${MYSQL_DB_CONFIG_PARAM} --all-databases --create-options --add-drop-database --events ${SCHEMA_ONLY} | ${BORG_CALL};
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
+			. "${DIR}/borg.backup.functions.close.sh" 1;
 			exit $_backup_error;
 		fi;
 	fi;
-	echo "Prune repository with keep${KEEP_INFO:1}";
+	if [ -z "${ONE_TIME_TAG}" ]; then
-	${BORG_PRUNE};
+		echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+		echo "Prune repository with keep${KEEP_INFO:1}";
+		${BORG_PRUNE};
+	fi;
 else
 	${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" |
 	while read db; do
-		echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+		echo "========[DB: ${db}]========================[${MODULE}]====================================>";
+		echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 		# exclude checks
 		include=0;
 		if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
@@ -174,8 +184,8 @@ else
 			# prepare borg calls
 			FILENAME="${db}-${schema_flag}-${DB_TYPE}_${DB_VERSION}_${DB_HOST}_${DB_PORT}.sql"
 			# backup set:
-			BACKUP_SET_PREFIX="${db}-"
+			BACKUP_SET_PREFIX="${MODULE},${db}-"
-			BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
+			BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 			# borg call
 			BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
 			BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
@@ -190,16 +200,25 @@ else
 				_backup_error=$?;
 				if [ $_backup_error -ne 0 ]; then
 					echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
+					. "${DIR}/borg.backup.functions.close.sh" 1;
 					exit $_backup_error;
 				fi;
 			fi;
-			echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
+			if [ -z "${ONE_TIME_TAG}" ]; then
-			${BORG_PRUNE};
+				echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+				echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
+				${BORG_PRUNE};
+			fi;
 		else
 			echo "- [E] ${db}";
 		fi;
 	done;
 fi;
+# run compact at the end if not a dry run
+if [ -z "${ONE_TIME_TAG}" ]; then
+	# if this is borg version >1.2 we need to run compact after prune
+	. "${DIR}/borg.backup.functions.compact.sh";
+fi;
 
 . "${DIR}/borg.backup.functions.close.sh";
 

borg.backup.pgsql.sh

@@ -10,7 +10,7 @@
 
 # set last edit date + time
 MODULE="pgsql"
-MODULE_VERSION="1.0.1";
+MODULE_VERSION="1.1.1";
 
 
 DIR="${BASH_SOURCE%/*}"
@@ -43,6 +43,7 @@ _PATH_PG_VERSION=${PG_VERSION};
 _backup_error=$?;
 if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then
 	echo "[! $(date +'%F %T')] Cannot get PostgreSQL server version: ${_backup_error}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit $_backup_error;
 fi;
 
@@ -58,6 +59,7 @@ if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
 		_PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//');
 		if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
 			echo "[! $(date +'%F %T')] PostgreSQL not found in any paths";
+			. "${DIR}/borg.backup.functions.close.sh" 1;
 			exit 1;
 		fi;
 	fi;
@@ -69,14 +71,17 @@ PG_DUMPALL=${PG_PATH}'pg_dumpall';
 # check that command are here
 if [ ! -f "${PG_PSQL}" ]; then
 	echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 if [ ! -f "${PG_DUMP}" ]; then
 	echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 if [ ! -f "${PG_DUMPALL}" ]; then
 	echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 
@@ -95,58 +100,70 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		SCHEMA_ONLY='-s';
 		schema_flag='schema';
 	fi;
-	echo "--- [all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	echo "--- [BACKUP: all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	# Filename
 	FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
 	# backup set:
-	BACKUP_SET_PREFIX="all-";
+	BACKUP_SET_PREFIX="${MODULE},all-";
-	BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
+	BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 	# borg call
 	BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
 	BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
 		echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}";
-		echo "${BORG_PRUNE}";
+		if [ -z "${ONE_TIME_TAG}" ]; then
+			echo "${BORG_PRUNE}";
+		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
 		$(${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL});
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
+			. "${DIR}/borg.backup.functions.close.sh" 1;
 			exit $_backup_error;
 		fi;
 	fi;
-	echo "Prune repository with keep${KEEP_INFO:1}";
+	if [ -z "${ONE_TIME_TAG}" ]; then
-	${BORG_PRUNE};
+		echo "--- [PRUNE : all databases: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+		echo "Prune repository with keep${KEEP_INFO:1}";
+		${BORG_PRUNE};
+	fi;
 else
 	# dump globals first
 	db="pg_globals";
 	schema_flag="data";
-	echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 	# Filename
 	FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
 	# backup set:
-	BACKUP_SET_PREFIX="${db}-";
+	BACKUP_SET_PREFIX="${MODULE},${db}-";
-	BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
+	BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 	# borg call
 	BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
 	BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
 		echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}";
-		echo "${BORG_PRUNE}";
+		if [ -z "${ONE_TIME_TAG}" ]; then
+			echo "${BORG_PRUNE}";
+		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
 		${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL};
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
+			. "${DIR}/borg.backup.functions.close.sh" 1;
 			exit $_backup_error;
 		fi;
 	fi;
-	echo "Prune repository with keep${KEEP_INFO:1}";
+	if [ -z "${ONE_TIME_TAG}" ]; then
-	${BORG_PRUNE};
+		echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+		echo "Prune repository with keep${KEEP_INFO:1}";
+		${BORG_PRUNE};
+	fi;
 
 	# get list of tables
 	for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do
@@ -154,7 +171,8 @@ else
 		owner=$(echo ${owner_db} | cut -d "," -f 1);
 		db=$(echo ${owner_db} | cut -d "," -f 2);
 		encoding=$(echo ${owner_db} | cut -d "," -f 3);
-		echo "--- [${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+		echo "========[DB: ${db}]========================[${MODULE}]====================================>";
+		echo "--- [BACKUP: ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 		include=0;
 		if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
 			while read incl_db; do
@@ -192,10 +210,10 @@ else
 			# Filename
 			# Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql
 			FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
-			# backup set:
-			BACKUP_SET_NAME="${db}-${schema_flag}-${BACKUP_SET}";
 			# PER db either data or schema
-			BACKUP_SET_PREFIX="${db}-";
+			BACKUP_SET_PREFIX="${MODULE},${db}-";
+			# backup set:
+			BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 			# borg call
 			BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
 			# borg prune
@@ -203,23 +221,34 @@ else
 			if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 				echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
 				echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}";
-				echo "${BORG_PRUNE}";
+				if [ -z "${ONE_TIME_TAG}" ]; then
+					echo "${BORG_PRUNE}";
+				fi;
 			fi;
 			if [ ${DRYRUN} -eq 0 ]; then
 				${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL};
 				_backup_error=$?;
 				if [ $_backup_error -ne 0 ]; then
 					echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
+					. "${DIR}/borg.backup.functions.close.sh" 1;
 					exit $_backup_error;
 				fi;
 			fi;
-			echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
+			if [ -z "${ONE_TIME_TAG}" ]; then
-			${BORG_PRUNE};
+				echo "--- [PRUNE : ${db}: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+				echo "Prune repository prefixed ${BACKUP_SET_PREFIX} with keep${KEEP_INFO:1}";
+				${BORG_PRUNE};
+			fi;
 		else
 			echo "- [E] ${db}";
 		fi;
 	done;
 fi;
+# run compact at the end if not a dry run
+if [ -z "${ONE_TIME_TAG}" ]; then
+	# if this is borg version >1.2 we need to run compact after prune
+	. "${DIR}/borg.backup.functions.compact.sh";
+fi;
 
 . "${DIR}/borg.backup.functions.close.sh";
 

borg.backup.zabbix.sh

@@ -3,7 +3,7 @@
 # Backup zabbix config and settings only
 
 MODULE="zabbix"
-MODULE_VERSION="1.0.1";
+MODULE_VERSION="1.1.1";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -24,7 +24,8 @@ if [ -z "${ZABBIX_DUMP_BIN}" ]; then
 fi;
 if [ ! -z "${ZABBIX_CONFIG}" ] && [ ! -f "${ZABBIX_CONFIG}" ]; then
 	echo "[! $(date +'%F %T')] Cannot find zabbix config: ${ZABBIX_CONFIG}";
-	exit;
+	. "${DIR}/borg.backup.functions.close.sh" 1;
+	exit 1;
 fi;
 if [ -f "${ZABBIX_CONFIG}" ]; then
 	OPT_ZABBIX_CONFIG="-z ${ZABBIX_CONFIG}";
@@ -34,10 +35,12 @@ if [ "${ZABBIX_DATABASE}" = "psql" ]; then
 fi;
 if [ "${ZABBIX_DATABASE}" != "psql" ] && [ "${ZABBIX_DATABASE}" != "mysql" ]; then
 	echo "[! $(date +'%F %T')] Zabbix dump must have database set to either psql or mysql";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 if [ ! -f "${ZABBIX_DUMP_BIN}" ]; then
 	echo "[! $(date +'%F %T')] Zabbix dump script could not be found: ${ZABBIX_DUMP_BIN}";
+	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
 # -i (ignore)/ -f (backup)
@@ -50,8 +53,8 @@ fi;
 # Filename
 FILENAME="zabbix-config.c.sql";
 # backup set:
-BACKUP_SET_PREFIX="zabbix-settings-";
+BACKUP_SET_PREFIX="${MODULE},settings-";
-BACKUP_SET_NAME="${BACKUP_SET_PREFIX}${BACKUP_SET}";
+BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${BACKUP_SET}";
 
 # borg call
 BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
@@ -61,13 +64,23 @@ if [ -z "${BACKUP_SET_PREFIX}" ]; then
 	BORG_PRUNE=$(echo "${BORG_PRUNE}" | sed -e 's/-P //');
 fi;
 
-echo "--- [zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+echo "--- [BACKUP: zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
 if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 	echo "${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}"
-	echo "${BORG_PRUNE}";
+	if [ -z "${ONE_TIME_TAG}" ]; then
+		echo "${BORG_PRUNE}";
+	fi;
 fi;
 if [ ${DRYRUN} -eq 0 ]; then
 	${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL};
 fi;
-echo "Prune repository with keep${KEEP_INFO:1}";
+if [ -z "${ONE_TIME_TAG}" ]; then
-${BORG_PRUNE};
+	echo "--- [PRUNE : $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+	${BORG_PRUNE};
+	# if this is borg version >1.2 we need to run compact after prune
+	. "${DIR}/borg.backup.functions.compact.sh";
+fi;
+
+. "${DIR}/borg.backup.functions.close.sh";
+
+# __EMD__