Borg backup update for gitea dump, mysql/postgresql info block

Gitea dump has now settings for
- temp folder (GITEA_TEMP_DIR)
- new name for working dir (GITEA_WORKING_DIR) from old (GITEA_TMP)

MySQL and PostgreSQL exports have a run time output per db dump set and not only at the end of the full run

.gitignore

@@ -6,3 +6,4 @@ borg.backup.*.include
 borg.backup.*.exclude
 borg.backup.*.schema-only
 borg.backup.*.init
+borg.backup.*.compact

borg.backup.file.sh

@@ -4,7 +4,7 @@
 
 # set last edit date + time
 MODULE="file";
-MODULE_VERSION="1.2.2";
+MODULE_VERSION="1.2.3";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi

borg.backup.functions.check.sh

@@ -59,15 +59,15 @@ if [ ${RUN_CHECK} -eq 1 ]; then
 	fi;
 	# debug/dryrun
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_REMOTE} ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY}";
 	fi;
 	# run info command if not a dry drun
 	if [ ${DRYRUN} -eq 0 ]; then
 		# if glob add glob command directly
 		if [[ "${CHECK_PREFIX}" =~ $REGEX_GLOB ]]; then
-		${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} -a "${CHECK_PREFIX}" ${REPOSITORY};
+		${BORG_COMMAND} check ${OPT_REMOTE} ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} -a "${CHECK_PREFIX}" ${REPOSITORY};
 		else
-			${BORG_COMMAND} check ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY};
+			${BORG_COMMAND} check ${OPT_REMOTE} ${OPT_PROGRESS} ${OPT_CHECK_VERIFY_DATA} ${OPT_GLOB} ${REPOSITORY};
 		fi;
 	fi;
 	# print additional info for use --repair command
@@ -75,7 +75,7 @@ if [ ${RUN_CHECK} -eq 1 ]; then
 	if [ ${VERBOSE} -eq 1 ] && [ ${CHECK} -eq 1 ]; then
 		echo "";
 		echo "In case of needed repair: "
-		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_PROGRESS} --repair ${OPT_GLOB} ${REPOSITORY}";
+		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";${BORG_COMMAND} check ${OPT_REMOTE} ${OPT_PROGRESS} --repair ${OPT_GLOB} ${REPOSITORY}";
 		echo "Before running repair, a copy from the backup should be made because repair might damage a backup"
 	fi;
 fi;

borg.backup.functions.compact.sh

@@ -46,7 +46,7 @@ if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
 		# reset to normal IFS, so command works here
 		IFS=${_IFS};
 		printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}";
-		BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}";
+		BORG_COMPACT="${BORG_COMMAND} compact ${OPT_REMOTE} -v ${OPT_PROGRESS} ${REPOSITORY}";
 		if [ ${DEBUG} -eq 1 ]; then
 				echo "${BORG_COMPACT}";
 		fi;

borg.backup.functions.init.sh

@@ -95,7 +95,8 @@ PRINTF_INFO_STRING="%-23s: %s\n";
 PRINTF_MASTER_BLOCK="=== [%-8s: %19s] ==[%s]====================================>\n";
 PRINTF_SUB_BLOCK="|-- [%-8s: %19s] --[%s]------------------------------------>\n";
 PRINTF_SUBEXT_BLOCK="|-- [%-8s: %s: %19s] --[%s]------------------------------------>\n";
-PRINTF_DB_SUB_BLOCK=">>- [%-8s: %s] =======================[%s]====================================>\n";
+PRINTF_DB_SUB_BLOCK=">>- [%-8s: %s] ==[%s]=======================>\n";
+PRINTF_DB_RUN_TIME_SUB_BLOCK=">>- [%-8s: %s] ==[%s]==[Run time: %s:]=======================>\n";
 # opt flags
 OPT_VERBOSE="";
 OPT_PROGRESS="";
@@ -413,6 +414,7 @@ elif [ ! -z "${BORG_EXECUTEABLE}" ]; then
 		exit;
 	fi;
 elif ! command -v borg &> /dev/null; then
+# elif [ -z ($command -v borg) ]; then
 	echo "borg backup seems not to be installed, please verify paths";
 	exit;
 fi;

borg.backup.functions.verify.sh

@@ -434,10 +434,10 @@ if [ ! -z "${DELETE_ONE_TIME_TAG}" ]; then
 	# not that compact only works on borg 1.2
 	if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
 		if [ ${DRYRUN} -eq 0 ]; then
-			${BORG_COMMAND} compact ${REPOSITORY};
+			${BORG_COMMAND} compact ${OPT_REMOTE} ${REPOSITORY};
 		fi;
 		if [ ${DEBUG} -eq 1 ]; then
-			echo "${BORG_COMMAND} compact ${REPOSITORY}";
+			echo "${BORG_COMMAND} compact ${OPT_REMOTE} ${REPOSITORY}";
 		fi;
 	fi;
 	. "${DIR}/borg.backup.functions.close.sh";

borg.backup.gitea.settings-default

@@ -2,10 +2,8 @@
 
 # rename to borg.backup.gitea.settings to use
 
-# override settings in borg.backup.settings with SUB_ prefix
-# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_*
-
 GIT_USER="";
-GITEA_TMP="";
+GITEA_WORKING_DIR="";
+GITEA_TEMP_DIR="";
 GITEA_BIN="";
 GITEA_CONFIG="";

borg.backup.gitea.sh

@@ -29,9 +29,13 @@ if [ -z "${GIT_USER}" ]; then
 	GIT_USER="git";
 fi;
 # set GITEA_* if not set
-if [ -z "${GITEA_TMP}" ]; then
+if [ -z "${GITEA_WORKING_DIR}" ]; then
 	# run gitea backup (user mktemp?)
-	GITEA_TMP="/tmp/gitea/";
+	GITEA_WORKING_DIR="/var/tmp/gitea/";
+fi;
+# general temp folder for temporary data storage, this is not working output folder
+if [ -z "${GITEA_TEMP_DIR}"]; then
+	GITEA_TEMP_DIR="/var/tmp";
 fi;
 if [ -z "${GITEA_BIN}" ]; then
 	GITEA_BIN="/usr/local/bin/gitea";
@@ -39,6 +43,10 @@ fi;
 if [ -z "${GITEA_CONFIG}" ]; then
 	GITEA_CONFIG="/etc/gitea/app.ini"
 fi;
+# This one is not advertised in the config file as it is not recommended to change
+if [ -z "${GITEA_EXPORT_TYPE}" ]; then
+	GITEA_EXPORT_TYPE="zip";
+fi;
 if [ ! -f "${GITEA_BIN}" ]; then
 	echo "[! $(date +'%F %T')] Cannot find gitea binary";
 	. "${DIR}/borg.backup.functions.close.sh" 1;
@@ -49,6 +57,16 @@ if [ ! -f "${GITEA_CONFIG}" ]; then
 	. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
+# some basic checks with abort
+if [ ! -d "${GITEA_TEMP_DIR}" ]; then
+	echo "Temp directory does not exist: ${GITEA_TEMP_DIR}";
+	exit;
+fi;
+# we should check GITEA_EXPORT_TYPE too at some point for an allow list
+# At the moment warn if not zip
+if [ "${GITEA_EXPORT_TYPE}" != "zip" ]; then
+	echo "[!!!!] The gitea export type has been changed from 'zip' to '${GITEA_EXPORT_TYPE}'. This can either break or make exports take very ling";
+fi;
 # Filename
 FILENAME="gitea.backup.zip";
 # backup set and prefix
@@ -60,7 +78,7 @@ BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e
 BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
 printf "${PRINTF_SUB_BLOCK}" "BACKUP: git data and database" "$(date +'%F %T')" "${MODULE}";
 if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
-	echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}";
+	echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_WORKING_DIR} -t ${GITEA_TEMP_DIR} --type ${GITEA_EXPORT_TYPE} -L -f - | ${BORG_CALL}";
 	if [ -z "${ONE_TIME_TAG}" ]; then
 		echo "${BORG_PRUNE}";
 	fi;
@@ -70,13 +88,13 @@ if [ ${DRYRUN} -eq 0 ]; then
 		# below was an old workaround
 		#export USER="${LOGNAME}" # workaround for broken gitea EUID check
 		# make sure temp folder is there and is set as git. user
-		if [ ! -d "${GITEA_TMP}" ]; then
-			mkdir -p "${GITEA_TMP}";
+		if [ ! -d "${GITEA_WORKING_DIR}" ]; then
+			mkdir -p "${GITEA_WORKING_DIR}";
 		fi;
-		chown -R ${GIT_USER}. "${GITEA_TMP}";
+		chown -R ${GIT_USER}. "${GITEA_WORKING_DIR}";
 		# this needs to be run in a folder that can be stat by git user
-		cd "${GITEA_TMP}";
-		sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL};
+		cd "${GITEA_WORKING_DIR}";
+		sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_WORKING_DIR} -t ${GITEA_TEMP_DIR} --type ${GITEA_EXPORT_TYPE} -L -f - | ${BORG_CALL};
 	) 2>&1 | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' # remove all ESC strings
 fi;
 if [ -z "${ONE_TIME_TAG}" ]; then

borg.backup.mysql.sh

@@ -93,6 +93,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		SCHEMA_ONLY='--no-data';
 		schema_flag='schema';
 	fi;
+	LOCAL_START=$(date +'%s');
 	printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}";
 	# We only do a full backup and not per table backup here
 	# Filename
@@ -124,9 +125,12 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		echo "Prune repository with keep${KEEP_INFO:1}";
 		${BORG_PRUNE};
 	fi;
+	DURATION=$[ $(date +'%s')-$LOCAL_START ];
+	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "databases" "${MODULE}" "$(convert_time ${DURATION})";
 else
 	${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" |
 	while read db; do
+		LOCAL_START=$(date +'%s');
 		printf "${PRINTF_DB_SUB_BLOCK}" "DB" "${db}" "${MODULE}";
 		printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}";
 		# exclude checks
@@ -213,6 +217,8 @@ else
 		else
 			echo "- [E] ${db}";
 		fi;
+		DURATION=$[ $(date +'%s')-$LOCAL_START ];
+		printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 	done;
 fi;
 # run compact at the end if not a dry run

borg.backup.pgsql.sh

@@ -102,6 +102,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		SCHEMA_ONLY='-s';
 		schema_flag='schema';
 	fi;
+	LOCAL_START=$(date +'%s');
 	printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}";
 	# Filename
 	FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
@@ -132,10 +133,13 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 		echo "Prune repository with keep${KEEP_INFO:1}";
 		${BORG_PRUNE};
 	fi;
+	DURATION=$[ $(date +'%s')-$LOCAL_START ];
+	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "databases" "${MODULE}" "$(convert_time ${DURATION})";
 else
 	# dump globals first
 	db="pg_globals";
 	schema_flag="data";
+	LOCAL_START=$(date +'%s');
 	printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}";
 	# Filename
 	FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
@@ -166,9 +170,11 @@ else
 		echo "Prune repository with keep${KEEP_INFO:1}";
 		${BORG_PRUNE};
 	fi;
+	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "BACKUP" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 
 	# get list of tables
 	for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do
+		LOCAL_START=$(date +'%s');
 		# get the user who owns the DB too
 		owner=$(echo ${owner_db} | cut -d "," -f 1);
 		db=$(echo ${owner_db} | cut -d "," -f 2);
@@ -265,6 +271,7 @@ else
 		else
 			echo "- [E] ${db}";
 		fi;
+		printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 	done;
 fi;
 # run compact at the end if not a dry run