Merge branch 'development' into shellcheck-cleanup

_borg_backup_set_prefix_cleanup.sh

@@ -31,16 +31,16 @@ MODULE_LIST="file gitea mysql pgsql zabbix"
 # -n for dry run test
 while getopts ":c:snd" opt; do
 	case "${opt}" in
-		c|config)
+		c) # config
 			BASE_FOLDER=${OPTARG};
 			;;
-		s|nosudo)
+		s) # nosudo
 			PGSQL_SUDO=0;
 			;;
-		d|debug)
+		d) # debug
 			DEBUG=1;
 			;;
-		n|dryrun)
+		n) # dryrun
 			DRYRUN=1;
 			;;
 		:)
@@ -64,6 +64,7 @@ if [ ! -f "${BASE_FOLDER}${SETTINGS_FILE}" ]; then
 	echo "Could not find: ${BASE_FOLDER}${SETTINGS_FILE}";
 	exit;
 fi;
+# shellcheck source="borg.backup.settings"
 . "${BASE_FOLDER}${SETTINGS_FILE}";
 
 if [ -n "${TARGET_BORG_PATH}" ]; then

borg.backup.functions.compact.sh

@@ -10,6 +10,7 @@ fi;
 
 # compact (only if BORG COMPACT is set)
 # only for borg 1.2
+if [ "$(version "$BORG_VERSION")" -ge "$(version "1.2.0")" ]; then
 if [ "$(version "$BORG_VERSION")" -ge "$(version "1.2.0")" ]; then
 	RUN_COMPACT=0;
 	if [ $# -ge 1 ] && [ "$1" = "auto" ]; then
@@ -21,10 +22,13 @@ if [ "$(version "$BORG_VERSION")" -ge "$(version "1.2.0")" ]; then
 		fi;
 		# get current date timestmap
 		CURRENT_DATE=$(date +%s);
+		if [ "${COMPACT_INTERVAL}" -eq 1 ]; then
 		if [ "${COMPACT_INTERVAL}" -eq 1 ]; then
 			RUN_COMPACT=1;
 			# set new compact time here
 			echo "${CURRENT_DATE}" > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}";
+		elif [ "${COMPACT_INTERVAL}" -gt 1 ]; then
+			echo "${CURRENT_DATE}" > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}";
 		elif [ "${COMPACT_INTERVAL}" -gt 1 ]; then
 			# else load last timestamp and check if today - last time stamp > days
 			if [ -z "${LAST_COMPACT_DATE}" ]; then
@@ -48,6 +52,7 @@ if [ "$(version "$BORG_VERSION")" -ge "$(version "1.2.0")" ]; then
 	if [ "${RUN_COMPACT}" -eq 1 ]; then
 		# reset to normal IFS, so command works here
 		IFS=${_IFS};
+		# shellcheck disable=SC2059
 		printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}";
 		BORG_COMPACT="${BORG_COMMAND} compact ${OPT_REMOTE} -v ${OPT_PROGRESS} ${REPOSITORY}";
 		if [ "${DEBUG}" -eq 1 ]; then

borg.backup.functions.init.sh

@@ -5,24 +5,16 @@ if [ -z "${MODULE}" ]; then
 	exit 1;
 fi;
 
-# E: inherit trap ERR
-# T: DEBUG and RETURN traps are inherited
-# u: unset variables ere error
 set -ETu #-e -o pipefail
-trap _catch ERR
-trap _cleanup SIGINT SIGTERM
+trap cleanup SIGINT SIGTERM ERR
 
-_cleanup() {
+cleanup() {
 	# script cleanup here
-	echo "Script abort: $? @LINE: $(caller)";
+	echo "Some part of the script failed with an error: $? @LINE: $(caller)";
 	# unset exported vars
 	unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK;
 	# end trap
-	trap - SIGINT SIGTERM
-}
-_catch() {
-	local last_exit_code=$?;
-	echo "Some part of the script failed with ERROR: $last_exit_code @COMMAND: '$BASH_COMMAND' @LINE: $(caller)" >&2;
+	trap - SIGINT SIGTERM ERR
 }
 # on exit unset any exported var
 trap "unset BORG_BASE_DIR BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK BORG_RELOCATED_REPO_ACCESS_IS_OK" EXIT;
@@ -32,7 +24,7 @@ function version {
 }
 
 # version for all general files
-VERSION="4.7.2";
+VERSION="4.5.4";
 
 # borg version and borg comamnd
 BORG_VERSION="";
@@ -93,7 +85,7 @@ REGEX="";
 REGEX_COMMENT="^[\ \t]*#";
 REGEX_GLOB='\*';
 REGEX_NUMERIC="^[0-9]{1,2}$";
-REGEX_ERROR="^Some part of the script failed with ERROR:";
+REGEX_ERROR="^Some part of the script failed with an error:";
 PRUNE_DEBUG="";
 INIT_REPOSITORY=0;
 FOLDER_OK=0;
@@ -356,7 +348,7 @@ if [ -n "${ONE_TIME_TAG}" ] && ! [[ "${ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+$ ]]; th
 	echo "One time tag '${ONE_TIME_TAG}' must be alphanumeric with dashes and underscore only.";
 	exit 1;
 elif [ -n "${ONE_TIME_TAG}" ]; then
-	# all ok, attach '.' at the end
+	# all ok, attach . at the end
 	ONE_TIME_TAG=${ONE_TIME_TAG}".";
 fi;
 # if -D, cannot be with -T, -i, -C, -I, -P
@@ -367,7 +359,7 @@ fi;
 # -D also must be in valid backup set format
 # ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]]
 if [ -n "${DELETE_ONE_TIME_TAG}" ] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},([A-Za-z0-9_-]+-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$ ]] && ! [[ "${DELETE_ONE_TIME_TAG}" =~ ^[A-Za-z0-9_-]+\.${MODULE},(\*-)?[0-9]{4}-[0-9]{2}-[0-9]{2}T\*$ ]]; then
-	echo "Delete one time tag '${DELETE_ONE_TIME_TAG}' is in an invalid format."
+	echo "Delete one time tag '${DELETE_ONE_TIME_TAG}' is in an invalid format. "
 	echo "Please verify existing tags with -P option."
 	echo "For a globing be sure it is in the format of: TAG.MODULE,*-YYYY-MM-DDT*";
 	echo "Note the dash (-) after the first *, also time (T) is a globa (*) must."

borg.backup.functions.verify.sh

@@ -294,15 +294,18 @@ COMMAND_INFO="${COMMAND_EXPORT}${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY}"
 # MARK: VERIFY / INFO
 if [ "${VERIFY}" -eq 1 ] || [ "${INIT}" -eq 1 ]; then
 	printf "${PRINTF_SUB_BLOCK}" "VERIFY" "$(date +'%F %T')" "${MODULE}";
-	if [ "${DEBUG}" -eq 1 ]; then
-		echo "${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1	";
-	fi;
-	# use borg info and verify if it returns "Repository ID:" in the first line
-	REPO_VERIFY=$(${BORG_COMMAND} info ${OPT_REMOTE} "${REPOSITORY}" 2>&1);
-	__last_error=$?;
-	# on any error in verify command force new INIT
-	if [[ $__last_error -ne 0 ]]; then
-		echo "[!] Repository verify error: ${REPO_VERIFY}";
+	if [ -n "${TARGET_SERVER}" ]; then
+		if [ "${DEBUG}" -eq 1 ]; then
+			echo "${BORG_COMMAND} info ${OPT_REMOTE} ${REPOSITORY} 2>&1|grep \"Repository ID:\"";
+		fi;
+		# use borg info and verify if it returns "Repository ID:" in the first line
+		REPO_VERIFY=$(${BORG_COMMAND} info ${OPT_REMOTE} "${REPOSITORY}" 2>&1|grep "Repository ID:");
+		# this is currently a hack to work round the error code in borg info
+		# this checks if REPO_VERIFY holds this error message and then starts init
+		if [[ -z "${REPO_VERIFY}" ]] || [[ "${REPO_VERIFY}" =~ ${REGEX_ERROR} ]]; then
+			INIT_REPOSITORY=1;
+		fi;
+	elif [ ! -d "${REPOSITORY}" ]; then
 		INIT_REPOSITORY=1;
 	fi;
 	# if verrify but no init and repo is there but init file is missing set it
@@ -359,7 +362,7 @@ if [ "${INIT}" -eq 1 ] && [ "${INIT_REPOSITORY}" -eq 1 ]; then
 	# exit after init
 	exit;
 elif [ "${INIT}" -eq 1 ] && [ "${INIT_REPOSITORY}" -eq 0 ]; then
-	echo "[!] ($(date +'%F %T')) Repository already initialized";
+	echo "[! $(date +'%F %T')] Repository already initialized";
 	echo "For more information run:"
 	echo "${COMMAND_INFO}";
 	. "${DIR}/borg.backup.functions.close.sh" 1;
@@ -368,7 +371,7 @@ fi;
 
 # verify for init file
 if [ ! -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then
-	echo "[!] ($(date +'%F %T')) It seems the repository has never been initialized."
+	echo "[! $(date +'%F %T')] It seems the repository has never been initialized."
 	echo "Please run -I to initialize or if already initialzed run with -C for init update."
 	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;

borg.backup.gitea.sh

@@ -6,7 +6,7 @@
 # Backup gitea database, all git folders and gitea settings
 
 MODULE="gitea"
-MODULE_VERSION="1.2.2";
+MODULE_VERSION="1.2.1";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi

borg.backup.mysql.sh

@@ -10,7 +10,7 @@
 
 # set last edit date + time
 MODULE="mysql"
-MODULE_VERSION="1.1.5";
+MODULE_VERSION="1.1.4";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi

borg.backup.pgsql.sh

@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
 
-# allow variables in printf format string
-# shellcheck disable=SC2059
-
 # Backup PostgreSQL
 # default is per table dump, can be set to one full dump
 # config override set in borg.backup.pgsql.settings
@@ -10,7 +7,7 @@
 
 # set last edit date + time
 MODULE="pgsql"
-MODULE_VERSION="1.2.8";
+MODULE_VERSION="1.2.7";
 
 
 DIR="${BASH_SOURCE%/*}"
@@ -123,7 +120,7 @@ if [ -n "${DATABASE_FULL_DUMP}" ]; then
 		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
-		${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL};
+		$(${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL});
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
@@ -136,7 +133,7 @@ if [ -n "${DATABASE_FULL_DUMP}" ]; then
 		echo "Prune repository with keep${KEEP_INFO:1}";
 		${BORG_PRUNE};
 	fi;
-	DURATION=$(( $(date +'%s') - LOCAL_START ));
+	DURATION=$[ $(date +'%s')-$LOCAL_START ];
 	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "all databases" "${MODULE}" "$(convert_time ${DURATION})";
 else
 	# dump globals first
@@ -173,21 +170,21 @@ else
 		echo "Prune repository with keep${KEEP_INFO:1}";
 		${BORG_PRUNE};
 	fi;
-	DURATION=$(( $(date +'%s') - LOCAL_START ));
+	DURATION=$[ $(date +'%s')-$LOCAL_START ];
 	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 
 	# get list of tables
-	for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"); do
+	for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do
 		LOCAL_START=$(date +'%s');
 		# get the user who owns the DB too
-		owner=$(echo "${owner_db}" | cut -d "," -f 1);
-		db=$(echo "${owner_db}" | cut -d "," -f 2);
-		encoding=$(echo "${owner_db}" | cut -d "," -f 3);
+		owner=$(echo ${owner_db} | cut -d "," -f 1);
+		db=$(echo ${owner_db} | cut -d "," -f 2);
+		encoding=$(echo ${owner_db} | cut -d "," -f 3);
 		printf "${PRINTF_DB_SUB_BLOCK}" "DB" "${db}" "${MODULE}";
 		printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}";
 		include=0;
 		if [ -s "${BASE_FOLDER}${INCLUDE_FILE}" ]; then
-			while read -r incl_db; do
+			while read incl_db; do
 				if [ "${db}" = "${incl_db}" ]; then
 					include=1;
 					break;
@@ -198,7 +195,7 @@ else
 		fi;
 		exclude=0;
 		if [ -f "${BASE_FOLDER}${EXCLUDE_FILE}" ]; then
-			while read -r excl_db; do
+			while read excl_db; do
 				if [ "${db}" = "${excl_db}" ]; then
 					exclude=1;
 					break;
@@ -214,7 +211,7 @@ else
 				# default is data dump
 				SCHEMA_ONLY='';
 				schema_flag='data';
-				while read -r schema_db; do
+				while read schema_db; do
 					if [ "${db}" = "${schema_db}" ]; then
 						SCHEMA_ONLY='-s';
 						schema_flag='schema';
@@ -226,7 +223,7 @@ else
 				# default to schema, unless in data list
 				SCHEMA_ONLY='-s';
 				schema_flag='schema';
-				while read -r data_db; do
+				while read data_db; do
 					if [ "${db}" = "${data_db}" ]; then
 						SCHEMA_ONLY='';
 						schema_flag='data';
@@ -259,7 +256,7 @@ else
 				fi;
 			fi;
 			if [ ${DRYRUN} -eq 0 ]; then
-				${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c "${db}" | ${BORG_CALL};
+				${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL};
 				_backup_error=$?;
 				if [ $_backup_error -ne 0 ]; then
 					echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
@@ -275,7 +272,7 @@ else
 		else
 			echo "- [E] ${db}";
 		fi;
-		DURATION=$(( $(date +'%s') - LOCAL_START ));
+		DURATION=$[ $(date +'%s')-$LOCAL_START ];
 		printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 	done;
 fi;

borg.backup.settings-default

@@ -13,19 +13,21 @@ TARGET_PORT="";
 TARGET_BORG_PATH="";
 # folder where the backup folder will be created
 TARGET_FOLDER="";
-# the backup file (folder) for this backup, avoid dynamic variables in here, must end with .borg
+# the backup file (folder) for this host $(hostname), must end with .borg
 BACKUP_FILE="";
-# compression settings (none, lz4, zstd, zlib, lzma)
-# empty is default zstd
+# compression settings (empty for none, lz4, zstd, zlib, lzma)
+# level, if empty then default, else number between 0 and 9, or 1 to 22 for zstd
+# default is zstd, 3
 COMPRESSION="";
-# compression level, if empty then default 3, else number between 0 and 9, or 1 to 22 for zstd
 COMPRESSION_LEVEL="";
 # encryption settings:
 # SHA-256: 'none', 'authenticated', 'repokey', 'keyfile'
 # BLAKE2b: 'authenticated-blake2', 'repokey-blake2', 'keyfile-blake2'
-# Note: none does not encrypt and is not recommended
+# Note: none does not encrypt
+# Blank passwords allowed for only key (if used, use keyfile)
 # Default is keyfile
-# Blank passwords allowed for only keyfile, else passwords have to be set via BORG_PASSPHRASE or BORG_PASSCOMMAND
+# passwords have to be set via BORG_PASSPHRASE or BORG_PASSCOMMAND
+# keyfile can have blank passwords
 # See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically
 ENCRYPTION=""
 # force repository verify, default is off, set to true for verify on every run

borg.backup.zabbix.sh

@@ -6,7 +6,7 @@
 # Backup zabbix config and settings only
 
 MODULE="zabbix"
-MODULE_VERSION="1.1.4";
+MODULE_VERSION="1.1.3";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -55,7 +55,7 @@ if [ ! -f "${ZABBIX_DUMP_BIN}" ]; then
 	exit 1;
 fi;
 # -i (ignore)/ -f (backup)
-if [ ! -z "${ZABBIX_UNKNOWN_TABLES}" ]; then
+if [ -n "${ZABBIX_UNKNOWN_TABLES}" ]; then
 	OPT_ZABBIX_UNKNOWN_TABLES="-f";
 else
 	OPT_ZABBIX_UNKNOWN_TABLES="-i";

borg.mount.sh

@@ -80,7 +80,7 @@ if [ ${UMOUNT} -eq 0 ]; then
 		. ${BASE_FOLDER}${SETTINGS_FILE}
 		# set the borg backup file base on the settings data
 		# if we have user/host then we build the ssh command
-		if [ ! -z "${TARGET_USER}" ] && [ ! -z "${TARGET_HOST}" ]; then
+		if [ -n "${TARGET_USER}" ] && [ -n "${TARGET_HOST}" ]; then
 			TARGET_SERVER=${TARGET_USER}"@"${TARGET_HOST}":";
 		fi;
 		REPOSITORY=${TARGET_SERVER}${TARGET_FOLDER}${BACKUP_FILE};
@@ -90,7 +90,7 @@ if [ ${UMOUNT} -eq 0 ]; then
 
 	# check that the repostiory exists
 	REPOSITORY_OK=0;
-	if [ ! -z "${TARGET_SERVER}" ]; then
+	if [ -n "${TARGET_SERVER}" ]; then
 		# remove trailing : for this
 		TARGET_SERVER=${TARGET_SERVER/:};
 		# use ssh command to check remote existense