Add compact auto run in intervals like check archive

Same settings like check archive in days. Auto set to 1 to run after
each prune. Can be controlled per module.

Readme.md

@@ -68,6 +68,9 @@ exit after running verify `-V`
 ### `-I`
 init repository (must be run first)
 
+### `-Z`
+run `borg compact` over given repository
+
 ### `-C`
 run `borg check` over given repository
 
@@ -119,6 +122,7 @@ All below have default values if not set in the main settings file
  * COMPRESSION_LEVEL: 3
  * ENCRYPTION: none
  * FORCE_VERIFY: false
+ * COMPACT_INTERVAL: 1
  * CHECK_INTERVAL: none
  * KEEP_LAST: 0
  * KEEP_HOURS: 0
@@ -131,6 +135,7 @@ All module settings files can have the following prefixed with `SUB_` to overrid
  * SUB_BACKUP_FILE
  * SUB_COMPRESSION
  * SUB_COMPRESSION_LEVEL
+ * SUB_COMPACT_INTERVAL
  * SUB_CHECK_INTERVAL
  * SUB_BACKUP_SET
  * SUB_KEEP_LAST
@@ -259,3 +264,21 @@ ZABBIX_UNKNOWN_TABLES | '' | if set, changed to -f (force)
 ### Control files
 
 There are no control files for zabbix settings backup
+
+
+## File connection
+
+Running any of the commands below
+- borg.backup.file.sh
+- borg.backup.gitea.sh
+- borg.backup.mysql.sh
+- borg.backup.pgsql.sh
+- borg.backup.zabbix.sh
+
+1) Run `borg.backup.functions.init.sh` (always)
+2) Run `borg.backup.functions.verify.sh` (always)
+3) (other code in "file" module)
+4) Run `borg.backup.functions.info.sh` (always)
+5) Run `borg.backup.functions.compact.sh` (not if one time tag)
+6) Run `borg.backup.functions.check.sh` (not if one time tag)
+7) Run `borg.backup.functions.close.sh` (always, can be called on error too)

borg.backup.file.sh

@@ -4,7 +4,7 @@
 
 # set last edit date + time
 MODULE="file";
-MODULE_VERSION="1.2.1";
+MODULE_VERSION="1.2.2";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -13,8 +13,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 # include and exclude file
 INCLUDE_FILE="borg.backup.${MODULE}.include";
 EXCLUDE_FILE="borg.backup.${MODULE}.exclude";
-# init verify and check file
+# init verify, compact and check file
 BACKUP_INIT_FILE="borg.backup.${MODULE}.init";
+BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact";
 BACKUP_CHECK_FILE="borg.backup.${MODULE}.check";
 # lock file
 BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
@@ -195,7 +196,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then
 	# for the IFS="#" to work we need to replace options spaces with exactly ONE #
 	$(echo "${COMMAND}" | sed -e 's/[ ][ ]*/#/g') 2>&1 || echo "[!] Borg prune aborted";
 	# if this is borg version >1.2 we need to run compact after prune
-	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.compact.sh" "auto";
 	# check in auto mode
 	. "${DIR}/borg.backup.functions.check.sh" "auto";
 else

borg.backup.functions.compact.sh

@@ -7,17 +7,52 @@ fi;
 
 # compact (only if BORG COMPACT is set)
 # only for borg 1.2
-
-# reset to normal IFS, so command works here
-IFS=${_IFS};
 if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
-	printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}";
-	BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}";
-	if [ ${DEBUG} -eq 1 ]; then
-			echo "${BORG_COMPACT}";
+	RUN_COMPACT=0;
+	if [ $# -ge 1 ] && [ "$1" = "auto" ]; then
+		# strip any spaces and convert to int
+		COMPACT_INTERVAL=$(echo "${COMPACT_INTERVAL}" | sed -e 's/ //g');
+		# not a valid compact interval, do no compact
+		if [ -z "${COMPACT_INTERVAL##*[!0-9]*}" ]; then
+			COMPACT_INTERVAL=0;
+		fi;
+		# get current date timestmap
+		CURRENT_DATE=$(date +%s);
+		if [ ${COMPACT_INTERVAL} -eq 1 ]; then
+			RUN_COMPACT=1;
+			# set new compact time here
+			echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}";
+		elif [ ${COMPACT_INTERVAL} -gt 1 ]; then
+			# else load last timestamp and check if today - last time stamp > days
+			if [ -z "${LAST_COMPACT_DATE}" ]; then
+				LAST_COMPACT_DATE=$(cat "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" 2>/dev/null | sed -e 's/ //g');
+			fi;
+			# file date is not a timestamp
+			if [ -z "${LAST_COMPACT_DATE##*[!0-9]*}" ]; then
+				LAST_COMPACT_DATE=0;
+			fi;
+			# if the difference greate than compact date, run. COMPACT INTERVAL is in days
+			if [ $(($CURRENT_DATE-$LAST_COMPACT_DATE)) -ge $((${COMPACT_INTERVAL}*86400)) ]; then
+				RUN_COMPACT=1;
+				# set new compact time here
+				echo ${CURRENT_DATE} > "${BASE_FOLDER}${BACKUP_COMPACT_FILE}";
+			fi;
+		fi;
+	elif [ ${COMPACT} -eq 1 ]; then
+		RUN_COMPACT=1;
 	fi;
-	if [ ${DRYRUN} -eq 0 ]; then
-		${BORG_COMPACT};
+
+	if [ ${RUN_COMPACT} -eq 1 ]; then
+		# reset to normal IFS, so command works here
+		IFS=${_IFS};
+		printf "${PRINTF_SUB_BLOCK}" "COMPACT" "$(date +'%F %T')" "${MODULE}";
+		BORG_COMPACT="${BORG_COMMAND} compact -v ${OPT_PROGRESS} ${REPOSITORY}";
+		if [ ${DEBUG} -eq 1 ]; then
+				echo "${BORG_COMPACT}";
+		fi;
+		if [ ${DRYRUN} -eq 0 ]; then
+			${BORG_COMPACT};
+		fi;
 	fi;
 fi;
 

borg.backup.functions.init.sh

@@ -24,7 +24,7 @@ function version {
 }
 
 # version for all general files
-VERSION="4.3.0";
+VERSION="4.4.0";
 
 # borg version and borg comamnd
 BORG_VERSION="";
@@ -49,7 +49,9 @@ EXCLUDE_FILE="";
 # backup folder initialzed verify
 BACKUP_INIT_FILE="";
 BACKUP_INIT_DATE="";
-# fiel with last check timestamp
+# file with last compact date
+BACKUP_COMPACT_FILE="";
+# file with last check timestamp
 BACKUP_CHECK_FILE="";
 # one time backup prefix tag, if set will use <tag>.<prefix>-Y-M-DTh:m:s type backup prefix
 ONE_TIME_TAG="";
@@ -65,6 +67,7 @@ INFO=0;
 VERIFY=0;
 CHECK=0;
 CHECK_VERIFY_DATA=0;
+COMPACT=0;
 INIT=0;
 EXIT=0;
 PRINT=0;
@@ -88,7 +91,7 @@ INIT_REPOSITORY=0;
 FOLDER_OK=0;
 TMP_EXCLUDE_FILE="";
 # printf strings
-PRINTF_INFO_STRING="%-21s: %s\n";
+PRINTF_INFO_STRING="%-23s: %s\n";
 PRINTF_MASTER_BLOCK="=== [%-8s: %19s] ==[%s]====================================>\n";
 PRINTF_SUB_BLOCK="|-- [%-8s: %19s] --[%s]------------------------------------>\n";
 PRINTF_SUBEXT_BLOCK="|-- [%-8s: %s: %19s] --[%s]------------------------------------>\n";
@@ -127,6 +130,11 @@ ENCRYPTION="";
 DEFAULT_FORCE_VERIFY="false";
 FORCE_VERIFY="";
 FORCE_CHECK=""; # Deprecated name, use FORCE_VERIFY
+# compact
+DEFAULT_COMPACT_INTERVAL="1";
+COMPACT_INTERVAL="";
+SUB_COMPACT_INTERVAL="";
+# check
 # default interval is none
 DEFAULT_CHECK_INTERVAL="";
 CHECK_INTERVAL="";
@@ -192,6 +200,7 @@ function usage()
 	-T <tag>: create one time stand alone backup prefixed with tag name
 	-D <tag backup set>: remove a tagged backup set, full name must be given
 	-b <borg executable>: override default path
+	-Z: run compress after prune/backup. Only for borg 1.2 or newer
 	-C: run borg check if repository is ok
 	-y: in combination with -C: add --verify-data
 	-p <archive prefix|glob>: in combinatio with -C: only check archives with prefix or glob
@@ -213,7 +222,7 @@ function usage()
 }
 
 # set options
-while getopts ":c:L:T:D:b:p:vldniCVeIPyh" opt; do
+while getopts ":c:L:T:D:b:p:vldniCVeIPyZh" opt; do
 	case "${opt}" in
 		c|config)
 			BASE_FOLDER=${OPTARG};
@@ -230,6 +239,10 @@ while getopts ":c:L:T:D:b:p:vldniCVeIPyh" opt; do
 		b|borg)
 			OPT_BORG_EXECUTEABLE=${OPTARG};
 			;;
+		Z|Compact)
+			# will run compact alone
+			COMPACT=1;
+			;;
 		C|Check)
 			# will run borg check
 			# alt modes --repository-only, --archives-only,
@@ -422,6 +435,10 @@ fi;
 if [ -z "${ENCRYPTION}" ]; then
 	ENCRYPTION="${DEFAULT_ENCRYPTION}";
 fi;
+# check interval override
+if [ -z "${COMPACT_INTERVAL}" ]; then
+	COMPACT_INTERVAL="${DEFAULT_COMPACT_INTERVAL}";
+fi;
 if [ -z "${CHECK_INTERVAL}" ]; then
 	CHECK_INTERVAL="${DEFAULT_CHECK_INTERVAL}";
 fi;
@@ -471,6 +488,10 @@ if [ -f "${BASE_FOLDER}${SETTINGS_FILE_SUB}" ]; then
 	if [ ! -z "${SUB_COMPRESSION_LEVEL}" ]; then
 		COMPRESSION_LEVEL=${SUB_COMPRESSION_LEVEL};
 	fi;
+	# compact interval override
+	if [ ! -z "${SUB_COMPACT_INTERVAL}" ]; then
+		COMPACT_INTERVAL="${SUB_COMPACT_INTERVAL}";
+	fi;
 	# override check interval
 	if [ ! -z "${SUB_CHECK_INTERVAL}" ]; then
 		CHECK_INTERVAL="${SUB_CHECK_INTERVAL}";
@@ -537,6 +558,7 @@ if [[ ${TARGET_FOLDER} =~ ^~\/ ]]; then
 	exit 1;
 fi
 
+# COMPACT_INTERVAL must be a number from -1 to 365
 # CHECK_INTERVAL must be a number from -1 to 365
 
 # log file set and check

borg.backup.functions.verify.sh

@@ -12,15 +12,6 @@ BACKUP_INIT_DATE='';
 if [ -f "${BASE_FOLDER}${BACKUP_INIT_FILE}" ]; then
 	BACKUP_INIT_DATE=$(printf '%(%c)T' $(cat "${BASE_FOLDER}${BACKUP_INIT_FILE}" 2>/dev/null));
 fi;
-# last check date if set
-BACKUP_LAST_CHECK_DATE='';
-LAST_CHECK_DATE='';
-CONVERT_TIME='';
-if [ -f "${BASE_FOLDER}${BACKUP_CHECK_FILE}" ]; then
-	LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null);
-	BACKUP_LAST_CHECK_DATE=$(printf '%(%c)T' ${LAST_CHECK_DATE});
-	CONVERT_TIME=$(convert_time $(($(date +%s)-${LAST_CHECK_DATE})));
-fi;
 # start logging from here
 exec &> >(tee -a "${LOG}");
 printf "${PRINTF_MASTER_BLOCK}" "START" "$(date +'%F %T')" "${MODULE}";
@@ -37,10 +28,31 @@ printf "${PRINTF_INFO_STRING}" "Hostname" "${HOSTNAME}";
 printf "${PRINTF_INFO_STRING}" "Base folder" "${BASE_FOLDER}";
 # Module init date (when init file was writen)
 printf "${PRINTF_INFO_STRING}" "Module init date" "${BACKUP_INIT_DATE}";
+# print last compact date if positive integer
+# only if borg > 1.2
+if [ $(version $BORG_VERSION) -ge $(version "1.2.0") ]; then
+	if [ "${COMPACT_INTERVAL##*[!0-9]*}" ]; then
+		printf "${PRINTF_INFO_STRING}" "Module compact interval" "${COMPACT_INTERVAL}";
+		if [ -f "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" ]; then
+			LAST_COMPACT_DATE=$(cat "${BASE_FOLDER}${BACKUP_COMPACT_FILE}" 2>/dev/null);
+			printf "${PRINTF_INFO_STRING}" "Module last compact" \
+				"$(printf '%(%c)T' ${LAST_COMPACT_DATE}) ($(convert_time $(($(date +%s)-${LAST_COMPACT_DATE}))) ago)";
+		else
+			printf "${PRINTF_INFO_STRING}" "Module last compact" "No compact run yet"
+		fi;
+	fi;
+fi;
 # print last check date if positive integer
 if [ "${CHECK_INTERVAL##*[!0-9]*}" ]; then
 	printf "${PRINTF_INFO_STRING}" "Module check interval" "${CHECK_INTERVAL}";
-	printf "${PRINTF_INFO_STRING}" "Module last check" "${BACKUP_LAST_CHECK_DATE} (${CONVERT_TIME} ago)";
+	# get last check date
+	if [ -f "${BASE_FOLDER}${BACKUP_CHECK_FILE}" ]; then
+		LAST_CHECK_DATE=$(cat "${BASE_FOLDER}${BACKUP_CHECK_FILE}" 2>/dev/null);
+		printf "${PRINTF_INFO_STRING}" "Module last check" \
+			"$(printf '%(%c)T' ${LAST_CHECK_DATE}) ($(convert_time $(($(date +%s)-${LAST_CHECK_DATE}))) ago)";
+	else
+		printf "${PRINTF_INFO_STRING}" "Module last check" "No check run yet";
+	fi;
 fi;
 
 # if force verify is true set VERIFY to 1 unless INFO is 1
@@ -376,6 +388,13 @@ if [ ${PRINT} -eq 1 ]; then
 	exit;
 fi;
 
+# run borg compact command and exit
+if [ ${COMPACT} -eq 1 ]; then
+	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.close.sh";
+	exit;
+fi;
+
 # run borg check command and exit
 if [ ${CHECK} -eq 1 ]; then
 	. "${DIR}/borg.backup.functions.check.sh";

borg.backup.gitea.sh

@@ -3,15 +3,16 @@
 # Backup gitea database, all git folders and gitea settings
 
 MODULE="gitea"
-MODULE_VERSION="1.1.3";
+MODULE_VERSION="1.1.4";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 # init system
 . "${DIR}/borg.backup.functions.init.sh";
 
-# init verify and check file
+# init verify, compact and check file
 BACKUP_INIT_FILE="borg.backup.${MODULE}.init";
+BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact";
 BACKUP_CHECK_FILE="borg.backup.${MODULE}.check";
 # lock file
 BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
@@ -83,7 +84,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then
 	echo "Prune repository with keep${KEEP_INFO:1}";
 	${BORG_PRUNE};
 	# if this is borg version >1.2 we need to run compact after prune
-	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.compact.sh" "auto";
 	# check in auto mode
 	. "${DIR}/borg.backup.functions.check.sh" "auto";
 fi;

borg.backup.mysql.sh

@@ -7,7 +7,7 @@
 
 # set last edit date + time
 MODULE="mysql"
-MODULE_VERSION="1.1.1";
+MODULE_VERSION="1.1.2";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -18,8 +18,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 INCLUDE_FILE="borg.backup.${MODULE}.include";
 EXCLUDE_FILE="borg.backup.${MODULE}.exclude";
 SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only";
-# init verify and check file
+# init verify, compact and check file
 BACKUP_INIT_FILE="borg.backup.${MODULE}.init";
+BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact";
 BACKUP_CHECK_FILE="borg.backup.${MODULE}.check";
 # lock file
 BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
@@ -217,7 +218,7 @@ fi;
 # run compact at the end if not a dry run
 if [ -z "${ONE_TIME_TAG}" ]; then
 	# if this is borg version >1.2 we need to run compact after prune
-	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.compact.sh" "auto";
 	# check in auto mode
 	. "${DIR}/borg.backup.functions.check.sh" "auto";
 fi;

borg.backup.pgsql.sh

@@ -7,7 +7,7 @@
 
 # set last edit date + time
 MODULE="pgsql"
-MODULE_VERSION="1.2.1";
+MODULE_VERSION="1.2.2";
 
 
 DIR="${BASH_SOURCE%/*}"
@@ -20,8 +20,9 @@ INCLUDE_FILE="borg.backup.${MODULE}.include";
 EXCLUDE_FILE="borg.backup.${MODULE}.exclude";
 SCHEMA_ONLY_FILE="borg.backup.${MODULE}.schema-only";
 DATA_ONLY_FILE="borg.backup.${MODULE}.data-only";
-# init verify and check file
+# init verify, compact and check file
 BACKUP_INIT_FILE="borg.backup.${MODULE}.init";
+BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact";
 BACKUP_CHECK_FILE="borg.backup.${MODULE}.check";
 # lock file
 BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
@@ -269,7 +270,7 @@ fi;
 # run compact at the end if not a dry run
 if [ -z "${ONE_TIME_TAG}" ]; then
 	# if this is borg version >1.2 we need to run compact after prune
-	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.compact.sh" "auto";
 	# check in auto mode
 	. "${DIR}/borg.backup.functions.check.sh" "auto";
 fi;

borg.backup.settings-default

@@ -27,8 +27,14 @@ COMPRESSION_LEVEL="";
 # Blank passwords allowed for only key (if used, use keyfile)
 # See: http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically
 ENCRYPTION="";
-# force repository verirfy, default is off, set to true for verify on every run
+# force repository verify, default is off, set to true for verify on every run
 FORCE_VERIFY="";
+# compact interval, only if using borg 1.2 or higher
+# after how many days to run compress on data
+# default is 1 for run after each prune
+# set to 0 or negative to turn off
+# or any other value for every n days
+COMPACT_INTERVAL="";
 # check interval, if 0 or negative number, no check will ever run
 # if empty fall back to default set
 # if set to 1 then every time the script runs

borg.backup.zabbix.sh

@@ -3,15 +3,16 @@
 # Backup zabbix config and settings only
 
 MODULE="zabbix"
-MODULE_VERSION="1.1.2";
+MODULE_VERSION="1.1.3";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 # init system
 . "${DIR}/borg.backup.functions.init.sh";
 
-# init verify and check file
+# init verify, compact and check file
 BACKUP_INIT_FILE="borg.backup.${MODULE}.init";
+BACKUP_COMPACT_FILE="borg.backup.${MODULE}.compact";
 BACKUP_CHECK_FILE="borg.backup.${MODULE}.check";
 # lock file
 BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
@@ -81,7 +82,7 @@ if [ -z "${ONE_TIME_TAG}" ]; then
 	printf "${PRINTF_SUB_BLOCK}" "PRUNE" "$(date +'%F %T')" "${MODULE}";
 	${BORG_PRUNE};
 	# if this is borg version >1.2 we need to run compact after prune
-	. "${DIR}/borg.backup.functions.compact.sh";
+	. "${DIR}/borg.backup.functions.compact.sh" "auto";
 	# check in auto mode
 	. "${DIR}/borg.backup.functions.check.sh" "auto";
 fi;