Zabbix settings backup, Readme update, base calls location move

New module for zabbix settings backup based on zabbix-dump

Move the _BORG_CALL and _BORG_PRUNE base calls to functions.check
instead of each module (they are all the same)

Update Readme file

Update default settings and remove KEEP so the original defaults are
used

Readme.md

@@ -4,25 +4,149 @@ These scripts are wrappers around the main borg backup scripts.
 
 Modules for plain file backup, mysql and postgresql backup exists.
 
+## Recommended setup
+
+git clone this repostory into the target folder:
+`git clone <repo> borg`
+
+And in there create the needed settings files.
+
+Now the core scripts can be updated with a simple
+`git pull`
+
+No settings files will be overwritten
+
 ## Basic Settings
 
+`borg.backup.settings`
+
+This file must be configured, without it the backup will not work
+
+The following must be set or checked:
+LOG_FOLDER: default `/var/log/borg.backup/`
+TARGET_FOLDER: must be set to a path where the backups can be written
+BACKUP_FILE: the folder inside the TARGET_FOLDER that is the target for borg. Must end with `.borg`
+
+
+Note: BACKUP_FILE is the base name. For all except file (current) a module suffix will be added:
+
+eg:
+`foo.name.borg` wil be `foor.name-mysql.borg` for mysql backups.
+
+If `FILE_REPOSITORY_COMPATIBLE` is set to `false` in the borg.backup.file.settings then the file borg name will have `-file` added too. Currently this is not added to stay compatible with older scripts
+
+All module settings files can have the following prefixed with `SUB_` to override master settings:
+ * SUB_BACKUP_FILE
+ * SUB_COMPRESSION
+ * SUB_COMPRESSION_LEVEL
+ * SUB_BACKUP_SET
+ * SUB_KEEP_LAST
+ * SUB_KEEP_HOURS
+ * SUB_KEEP_DAYS
+ * SUB_KEEP_WEEKS
+ * SUB_KEEP_MONTHS
+ * SUB_KEEP_YEARS
+ * SUB_KEEP_WITHIN
+
+## Setup backup via SSH to remote host
+
+For this the following settings are from interest
+
+```
+TARGET_USER="";
+TARGET_HOST="";
+TARGET_PORT="";
+```
+
+Note that if `.ssh/config` is used only `TARGET_HOST` needs to be set. Recommened for handling proxy jumps and key files.
+
+and `TARGET_BORG_PATH="";` if the target borg is in a non default path
+
 ## File backup settings
 
+### Config variables
+
+
+### Control files
+
+```
+backup.borg.file.include
+backup.borg.file.exclude
+```
+
+`backup.borg.file.include` must be set
+
+### File content rules
+
 ## PostgreSQL backup settings
 
-DATABASE_FULL_DUMP="";
-DATABASE_USER="";
+This script must be run as the postgres user, normaly `postgres`.
+The postgres user must be added to the backup group for this, so that the basic init file can be created in the borg base folder.
+
+### Config variables
+
+Variable | Default | Description
+| - | - | - |
+DATABASE_FULL_DUMP | | if empty, dump per databse, if set dump all in one file, if set to schema dump only schema
+DATABASE_USER | | overide username to connect to database
+
+### Control files
+
+```
+backup.borg.pgsql.include
+backup.borg.pgsql.exclude
+backup.borg.pgsql.schema-only
+```
 
 ## MySQL backup settings
 
-DATABASE_FULL_DUMP="";
-MYSQL_DB_CONFIG="";
+If non root ident authentication run is used, be sure that the `mysql` user is in the backup group.
+
+### Config variables
+
+Variable | Default | Description
+| - | - | - |
+DATABASE_FULL_DUMP | | if empty, dump per databse, if set dump all in one file, if set to schema dump only schema
+MYSQL_DB_CONFIG | | override file for connection. In modern mariaDB installations it is rcommended to run the script as root or mysql user and use the ident authentication instead.
+
+### Control files
+
+```
+backup.borg.mysql.include
+backup.borg.mysql.exclude
+backup.borg.mysql.schema-only
+```
 
 ## gitea backup settings
 
-```
-GIT_USER="";
-GITEA_TMP="";
-GITEA_BIN="";
-GITEA_CONFIG="";
-```
+Note that the backup needs the GIT_USER set that runs gitea.
+This user is neede to create the temporary dump folder and access for the git files and database.
+
+### Config Variables
+
+Variable | Default | Description
+| - | - | - |
+GIT_USER | git | The user that runs gitea |
+GITEA_TMP | /tmp/gitea/ | Where the temporary dump files from the backup are stored, as user git |
+GITEA_BIN | /usr/local/bin/gitea | Where the gitea binary is located |
+GITEA_CONFIG | /etc/gitea/app.ini | The configuration file for gitea |
+
+
+### Control files
+
+There are no control files for gitea backup
+
+## zabbix config backup settings
+
+### Config Variables
+
+Variable | Default | Description
+| - | - | - |
+ZABBIX_DUMP | /usr/local/bin/zabbix-dump |
+ZABBIX_DATABASE | '' | Must be set as either psql or mysql
+ZABBIX_CONFIG | '' | if not set uses default location
+ZABBIX_UNKNOWN_TABLES | '' | if set, changed to -f (force)
+
+### Control files
+
+There are no control files for zabbix settings backup

borg.backup.functions.check.sh

@@ -194,6 +194,12 @@ IFS="#";
 if [ "${MODULE}" != "file" ]; then
 	IFS=${_IFS};
 fi;
+
+# borg call, replace ##...## parts during run
+# used in all modules, except 'file'
+_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
+_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
+
 # general borg settings
 # set base path to config directory to keep cache/config separated
 export BORG_BASE_DIR="${BASE_FOLDER}";

borg.backup.functions.init.sh

@@ -99,6 +99,14 @@ GIT_USER="";
 GITEA_TMP="";
 GITEA_BIN="";
 GITEA_CONFIG="";
+# zabbix module
+ZABBIX_DUMP_BIN="";
+ZABBIX_CONFIG="";
+ZABBIX_DATABASE="";
+ZABBIX_UNKNOWN_TABLES="";
+OPT_ZABBIX_DUMP="";
+OPT_ZABBIX_CONFIG="";
+OPT_ZABBIX_UNKNOWN_TABLES="";
 # default keep 7 days, 4 weeks, 6 months
 # if set 0, ignore
 # note that for last/hourly it is needed to create a different
@@ -398,7 +406,7 @@ function convert_time
 			time_string=${time_string}${output[$i]}${timenames[$i]};
 		fi;
 	done;
-	if [ ! -z ${ms} ] && [ ${ms} -gt 0 ]; then
+	if [ ! -z ${ms} ] && [ "${ms}" != "nan" ]; && [ ${ms} -gt 0 ]; then
 		time_string=${time_string}" "${ms}"ms";
 	fi;
 	# just in case the time is 0

borg.backup.gitea.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 MODULE="gitea"
-MODULE_VERSION="0.1.0";
+MODULE_VERSION="1.0.0";
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
@@ -48,8 +48,8 @@ BACKUP_SET_NAME="gitea-${BACKUP_SET}";
 BACKUP_SET_PREFIX="gitea-";
 
 # borg call, replace ##...## parts
-_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
-_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
+# _BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
+# _BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
 # borg call
 BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET##/${BACKUP_SET}/");
 BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");

borg.backup.mysql.sh

@@ -81,10 +81,6 @@ NOLOCKS="--single-transaction"
 EVENTDB="mysql"
 EVENTS="--events"
 
-# borg call, replace ##...## parts
-_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
-_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
-
 # ALL IN ONE FILE or PER DATABASE FLAG
 if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 	SCHEMA_ONLY='';

borg.backup.pgsql.sh

@@ -87,10 +87,6 @@ DB_HOST='local'; # or <host>
 CONN_DB_HOST=''; # -h <host>
 CONN_DB_PORT=''; # -p <port>
 
-# borg call, replace ##...## parts
-_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -";
-_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}";
-
 # ALL IN ONE FILE or PER DATABASE FLAG
 if [ ! -z "${DATABASE_FULL_DUMP}" ]; then
 	SCHEMA_ONLY='';

borg.backup.settings-default

@@ -34,7 +34,7 @@ FORCE_CHECK="";
 # it can also be "{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f}"
 BACKUP_SET="";
 # prune times, how many are kept in each time frame
-KEEP_DAYS=7;
-KEEP_WEEKS=4;
-KEEP_MONTHS=6;
-KEEP_YEARS=1;
+KEEP_DAYS="";
+KEEP_WEEKS="";
+KEEP_MONTHS="";
+KEEP_YEARS="";

borg.backup.zabbix.settings-default

@@ -0,0 +1,17 @@
+# Borg backup wrapper scripts settings: gitea
+
+# rename to borg.backup.gitea.settings to use
+
+# override settings in borg.backup.settings with SUB_ prefix
+# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_*
+
+# location for the zabbix dump script
+# Source: https://github.com/npotorino/zabbix-backup
+# if not set use /usr/local/bin
+ZABBIX_DUMP="";
+# either psql or mysql, must be set
+ZABBIX_DATABASE="";
+# config location (default /etc/zabbiz/zabbix_server.conf)
+ZABBIX_CONFIG="";
+# unknown tables, default ignore, is set to true, will force backup them
+ZABBIX_UNKNOWN_TABLES="";

borg.backup.zabbix.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+MODULE="zabbix"
+MODULE_VERSION="0.1.0";
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+# init system
+. "${DIR}/borg.backup.functions.init.sh";
+
+# init check file
+BACKUP_INIT_CHECK="borg.backup.zabbix.init";
+
+# check valid data
+. "${DIR}/borg.backup.functions.check.sh";
+# if info print info and then abort run
+. "${DIR}/borg.backup.functions.info.sh";
+
+# /usr/local/scripts/zabbix/zabbix-dump
+if [ -z "${ZABBIX_DUMP_BIN}" ]; then
+	ZABBIX_DUMP_BIN="/usr/local/bin/zabbix-dump";
+fi;
+if [ ! -z "${ZABBIX_CONFIG}" ] && [ ! -f "${ZABBIX_CONFIG}" ]; then
+	echo "[! $(date +'%F %T')] Cannot find zabbix config: ${ZABBIX_CONFIG}";
+	exit;
+fi;
+if [ -f "${ZABBIX_CONFIG}" ]; then
+	OPT_ZABBIX_CONFIG="-z ${ZABBIX_CONFIG}";
+fi;
+if [ "${ZABBIX_DATABASE}" = "psql" ]; then
+	OPT_ZABBIX_DUMP="-C";
+fi;
+if [ "${ZABBIX_DATABASE}" != 'psql' ] || [ "${ZABBIX_DATABASE}" != "mysql" ]; then
+	echo "[! $(date +'%F %T')] Zabbix dump must have database set to either psql or mysql";
+	exit 1;
+fi;
+if [ ! -f "${ZABBIX_DUMP}" ]; then
+	echo "[! $(date +'%F %T')] Zabbix dump script could not be found: ${ZABBIX_DUMP}";
+	exit 1;
+fi;
+# -i (ignore)/ -f (backup)
+if [ ! -z "${ZABBIX_UNKNOWN_TABLES}" ]; then
+	OPT_ZABBIX_UNKNOWN_TABLES="-f";
+else
+	OPT_ZABBIX_UNKNOWN_TABLES="-i";
+fi;
+
+# Filename
+FILENAME="zabbix-config.c.sql";
+# backup set:
+BACKUP_SET="zabbix-settings-${BACKUP_SET}";
+BACKUP_SET_PREFIX="zabbix-settings-";
+
+# borg call
+BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET##/${BACKUP_SET}/");
+BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
+# if prefix is emtpy remote "-P"
+if [ -z "${BACKUP_SET_PREFIX}" ]; then
+	BORG_PRUNE=$(echo "${BORG_PRUNE}" | sed -e 's/-P //');
+fi;
+
+echo "--- [zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>";
+if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
+	echo "${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}"
+	echo "${BORG_PRUNE}";
+fi;
+if [ ${DRYRUN} -eq 0 ]; then
+	${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL};
+fi;
+echo "Prune repository with keep${KEEP_INFO:1}";
+${BORG_PRUNE};