diff --git a/Readme.md b/Readme.md index a7d7d86..a92e554 100644 --- a/Readme.md +++ b/Readme.md @@ -4,25 +4,149 @@ These scripts are wrappers around the main borg backup scripts. Modules for plain file backup, mysql and postgresql backup exists. +## Recommended setup + +git clone this repostory into the target folder: +`git clone borg` + +And in there create the needed settings files. + +Now the core scripts can be updated with a simple +`git pull` + +No settings files will be overwritten + ## Basic Settings +`borg.backup.settings` + +This file must be configured, without it the backup will not work + +The following must be set or checked: +LOG_FOLDER: default `/var/log/borg.backup/` +TARGET_FOLDER: must be set to a path where the backups can be written +BACKUP_FILE: the folder inside the TARGET_FOLDER that is the target for borg. Must end with `.borg` + + +Note: BACKUP_FILE is the base name. For all except file (current) a module suffix will be added: + +eg: +`foo.name.borg` wil be `foor.name-mysql.borg` for mysql backups. + +If `FILE_REPOSITORY_COMPATIBLE` is set to `false` in the borg.backup.file.settings then the file borg name will have `-file` added too. Currently this is not added to stay compatible with older scripts + +All module settings files can have the following prefixed with `SUB_` to override master settings: + * SUB_BACKUP_FILE + * SUB_COMPRESSION + * SUB_COMPRESSION_LEVEL + * SUB_BACKUP_SET + * SUB_KEEP_LAST + * SUB_KEEP_HOURS + * SUB_KEEP_DAYS + * SUB_KEEP_WEEKS + * SUB_KEEP_MONTHS + * SUB_KEEP_YEARS + * SUB_KEEP_WITHIN + +## Setup backup via SSH to remote host + +For this the following settings are from interest + +``` +TARGET_USER=""; +TARGET_HOST=""; +TARGET_PORT=""; +``` + +Note that if `.ssh/config` is used only `TARGET_HOST` needs to be set. Recommened for handling proxy jumps and key files. + +and `TARGET_BORG_PATH="";` if the target borg is in a non default path + ## File backup settings +### Config variables + + +### Control files + +``` +backup.borg.file.include +backup.borg.file.exclude +``` + +`backup.borg.file.include` must be set + +### File content rules + ## PostgreSQL backup settings -DATABASE_FULL_DUMP=""; -DATABASE_USER=""; +This script must be run as the postgres user, normaly `postgres`. +The postgres user must be added to the backup group for this, so that the basic init file can be created in the borg base folder. + +### Config variables + +Variable | Default | Description +| - | - | - | +DATABASE_FULL_DUMP | | if empty, dump per databse, if set dump all in one file, if set to schema dump only schema +DATABASE_USER | | overide username to connect to database + +### Control files + +``` +backup.borg.pgsql.include +backup.borg.pgsql.exclude +backup.borg.pgsql.schema-only +``` ## MySQL backup settings -DATABASE_FULL_DUMP=""; -MYSQL_DB_CONFIG=""; +If non root ident authentication run is used, be sure that the `mysql` user is in the backup group. + +### Config variables + +Variable | Default | Description +| - | - | - | +DATABASE_FULL_DUMP | | if empty, dump per databse, if set dump all in one file, if set to schema dump only schema +MYSQL_DB_CONFIG | | override file for connection. In modern mariaDB installations it is rcommended to run the script as root or mysql user and use the ident authentication instead. + +### Control files + +``` +backup.borg.mysql.include +backup.borg.mysql.exclude +backup.borg.mysql.schema-only +``` ## gitea backup settings -``` -GIT_USER=""; -GITEA_TMP=""; -GITEA_BIN=""; -GITEA_CONFIG=""; -``` +Note that the backup needs the GIT_USER set that runs gitea. +This user is neede to create the temporary dump folder and access for the git files and database. + +### Config Variables + +Variable | Default | Description +| - | - | - | +GIT_USER | git | The user that runs gitea | +GITEA_TMP | /tmp/gitea/ | Where the temporary dump files from the backup are stored, as user git | +GITEA_BIN | /usr/local/bin/gitea | Where the gitea binary is located | +GITEA_CONFIG | /etc/gitea/app.ini | The configuration file for gitea | + + +### Control files + +There are no control files for gitea backup + +## zabbix config backup settings + +### Config Variables + +Variable | Default | Description +| - | - | - | +ZABBIX_DUMP | /usr/local/bin/zabbix-dump | +ZABBIX_DATABASE | '' | Must be set as either psql or mysql +ZABBIX_CONFIG | '' | if not set uses default location +ZABBIX_UNKNOWN_TABLES | '' | if set, changed to -f (force) + +### Control files + +There are no control files for zabbix settings backup diff --git a/borg.backup.functions.check.sh b/borg.backup.functions.check.sh index e1296e2..1e89ab0 100644 --- a/borg.backup.functions.check.sh +++ b/borg.backup.functions.check.sh @@ -194,6 +194,12 @@ IFS="#"; if [ "${MODULE}" != "file" ]; then IFS=${_IFS}; fi; + +# borg call, replace ##...## parts during run +# used in all modules, except 'file' +_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; +_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; + # general borg settings # set base path to config directory to keep cache/config separated export BORG_BASE_DIR="${BASE_FOLDER}"; diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index c271cc3..f04a0c5 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -99,6 +99,14 @@ GIT_USER=""; GITEA_TMP=""; GITEA_BIN=""; GITEA_CONFIG=""; +# zabbix module +ZABBIX_DUMP_BIN=""; +ZABBIX_CONFIG=""; +ZABBIX_DATABASE=""; +ZABBIX_UNKNOWN_TABLES=""; +OPT_ZABBIX_DUMP=""; +OPT_ZABBIX_CONFIG=""; +OPT_ZABBIX_UNKNOWN_TABLES=""; # default keep 7 days, 4 weeks, 6 months # if set 0, ignore # note that for last/hourly it is needed to create a different @@ -398,7 +406,7 @@ function convert_time time_string=${time_string}${output[$i]}${timenames[$i]}; fi; done; - if [ ! -z ${ms} ] && [ ${ms} -gt 0 ]; then + if [ ! -z ${ms} ] && [ "${ms}" != "nan" ]; && [ ${ms} -gt 0 ]; then time_string=${time_string}" "${ms}"ms"; fi; # just in case the time is 0 diff --git a/borg.backup.gitea.sh b/borg.backup.gitea.sh index cdc21aa..f00ae08 100755 --- a/borg.backup.gitea.sh +++ b/borg.backup.gitea.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash MODULE="gitea" -MODULE_VERSION="0.1.0"; +MODULE_VERSION="1.0.0"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi @@ -48,8 +48,8 @@ BACKUP_SET_NAME="gitea-${BACKUP_SET}"; BACKUP_SET_PREFIX="gitea-"; # borg call, replace ##...## parts -_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; -_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; +# _BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; +# _BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; # borg call BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET##/${BACKUP_SET}/"); BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh index 55fc75a..d0e2c47 100755 --- a/borg.backup.mysql.sh +++ b/borg.backup.mysql.sh @@ -81,10 +81,6 @@ NOLOCKS="--single-transaction" EVENTDB="mysql" EVENTS="--events" -# borg call, replace ##...## parts -_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; -_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; - # ALL IN ONE FILE or PER DATABASE FLAG if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY=''; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index 93b3235..032bf66 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -87,10 +87,6 @@ DB_HOST='local'; # or CONN_DB_HOST=''; # -h CONN_DB_PORT=''; # -p -# borg call, replace ##...## parts -_BORG_CALL="borg create ${OPT_REMOTE} -v ${OPT_LIST} ${OPT_PROGRESS} ${OPT_COMPRESSION} -s --stdin-name ##FILENAME## ${REPOSITORY}::##BACKUP_SET## -"; -_BORG_PRUNE="borg prune ${OPT_REMOTE} -v -s --list ${PRUNE_DEBUG} -P ##BACKUP_SET_PREFIX## ${KEEP_OPTIONS[*]} ${REPOSITORY}"; - # ALL IN ONE FILE or PER DATABASE FLAG if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY=''; diff --git a/borg.backup.settings-default b/borg.backup.settings-default index aa3924c..f435336 100644 --- a/borg.backup.settings-default +++ b/borg.backup.settings-default @@ -34,7 +34,7 @@ FORCE_CHECK=""; # it can also be "{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f}" BACKUP_SET=""; # prune times, how many are kept in each time frame -KEEP_DAYS=7; -KEEP_WEEKS=4; -KEEP_MONTHS=6; -KEEP_YEARS=1; +KEEP_DAYS=""; +KEEP_WEEKS=""; +KEEP_MONTHS=""; +KEEP_YEARS=""; diff --git a/borg.backup.zabbix.settings-default b/borg.backup.zabbix.settings-default new file mode 100644 index 0000000..3847cef --- /dev/null +++ b/borg.backup.zabbix.settings-default @@ -0,0 +1,17 @@ +# Borg backup wrapper scripts settings: gitea + +# rename to borg.backup.gitea.settings to use + +# override settings in borg.backup.settings with SUB_ prefix +# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_* + +# location for the zabbix dump script +# Source: https://github.com/npotorino/zabbix-backup +# if not set use /usr/local/bin +ZABBIX_DUMP=""; +# either psql or mysql, must be set +ZABBIX_DATABASE=""; +# config location (default /etc/zabbiz/zabbix_server.conf) +ZABBIX_CONFIG=""; +# unknown tables, default ignore, is set to true, will force backup them +ZABBIX_UNKNOWN_TABLES=""; diff --git a/borg.backup.zabbix.sh b/borg.backup.zabbix.sh new file mode 100644 index 0000000..81fc771 --- /dev/null +++ b/borg.backup.zabbix.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +MODULE="zabbix" +MODULE_VERSION="0.1.0"; + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +# init system +. "${DIR}/borg.backup.functions.init.sh"; + +# init check file +BACKUP_INIT_CHECK="borg.backup.zabbix.init"; + +# check valid data +. "${DIR}/borg.backup.functions.check.sh"; +# if info print info and then abort run +. "${DIR}/borg.backup.functions.info.sh"; + +# /usr/local/scripts/zabbix/zabbix-dump +if [ -z "${ZABBIX_DUMP_BIN}" ]; then + ZABBIX_DUMP_BIN="/usr/local/bin/zabbix-dump"; +fi; +if [ ! -z "${ZABBIX_CONFIG}" ] && [ ! -f "${ZABBIX_CONFIG}" ]; then + echo "[! $(date +'%F %T')] Cannot find zabbix config: ${ZABBIX_CONFIG}"; + exit; +fi; +if [ -f "${ZABBIX_CONFIG}" ]; then + OPT_ZABBIX_CONFIG="-z ${ZABBIX_CONFIG}"; +fi; +if [ "${ZABBIX_DATABASE}" = "psql" ]; then + OPT_ZABBIX_DUMP="-C"; +fi; +if [ "${ZABBIX_DATABASE}" != 'psql' ] || [ "${ZABBIX_DATABASE}" != "mysql" ]; then + echo "[! $(date +'%F %T')] Zabbix dump must have database set to either psql or mysql"; + exit 1; +fi; +if [ ! -f "${ZABBIX_DUMP}" ]; then + echo "[! $(date +'%F %T')] Zabbix dump script could not be found: ${ZABBIX_DUMP}"; + exit 1; +fi; +# -i (ignore)/ -f (backup) +if [ ! -z "${ZABBIX_UNKNOWN_TABLES}" ]; then + OPT_ZABBIX_UNKNOWN_TABLES="-f"; +else + OPT_ZABBIX_UNKNOWN_TABLES="-i"; +fi; + +# Filename +FILENAME="zabbix-config.c.sql"; +# backup set: +BACKUP_SET="zabbix-settings-${BACKUP_SET}"; +BACKUP_SET_PREFIX="zabbix-settings-"; + +# borg call +BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET##/${BACKUP_SET}/"); +BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s|##REPOSITORY##|${REPOSITORY}|" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); +# if prefix is emtpy remote "-P" +if [ -z "${BACKUP_SET_PREFIX}" ]; then + BORG_PRUNE=$(echo "${BORG_PRUNE}" | sed -e 's/-P //'); +fi; + +echo "--- [zabbix settings: $(date +'%F %T')] --[${MODULE}]------------------------------------>"; +if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then + echo "${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}" + echo "${BORG_PRUNE}"; +fi; +if [ ${DRYRUN} -eq 0 ]; then + ${ZABBIX_DUMP_BIN} -t ${ZABBIX_DATABASE} ${OPT_ZABBIX_UNKNOWN_TABLES} ${OPT_ZABBIX_DUMP} ${OPT_ZABBIX_CONFIG} -o - | ${BORG_CALL}; +fi; +echo "Prune repository with keep${KEEP_INFO:1}"; +${BORG_PRUNE};