diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index 5426fba..927b66f 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -95,7 +95,8 @@ PRINTF_INFO_STRING="%-23s: %s\n"; PRINTF_MASTER_BLOCK="=== [%-8s: %19s] ==[%s]====================================>\n"; PRINTF_SUB_BLOCK="|-- [%-8s: %19s] --[%s]------------------------------------>\n"; PRINTF_SUBEXT_BLOCK="|-- [%-8s: %s: %19s] --[%s]------------------------------------>\n"; -PRINTF_DB_SUB_BLOCK=">>- [%-8s: %s] =======================[%s]====================================>\n"; +PRINTF_DB_SUB_BLOCK=">>- [%-8s: %s] ==[%s]=======================>\n"; +PRINTF_DB_RUN_TIME_SUB_BLOCK=">>- [%-8s: %s] ==[%s]==[Run time: %s:]=======================>\n"; # opt flags OPT_VERBOSE=""; OPT_PROGRESS=""; diff --git a/borg.backup.gitea.settings-default b/borg.backup.gitea.settings-default index 8a2f098..079b869 100644 --- a/borg.backup.gitea.settings-default +++ b/borg.backup.gitea.settings-default @@ -2,10 +2,8 @@ # rename to borg.backup.gitea.settings to use -# override settings in borg.backup.settings with SUB_ prefix -# valid for BACKUP_FILE, BACKUP_SET, COMPRESSION*, KEEP_* - GIT_USER=""; -GITEA_TMP=""; +GITEA_WORKING_DIR=""; +GITEA_TEMP_DIR=""; GITEA_BIN=""; GITEA_CONFIG=""; diff --git a/borg.backup.gitea.sh b/borg.backup.gitea.sh index bca7bbe..49ff567 100755 --- a/borg.backup.gitea.sh +++ b/borg.backup.gitea.sh @@ -29,9 +29,13 @@ if [ -z "${GIT_USER}" ]; then GIT_USER="git"; fi; # set GITEA_* if not set -if [ -z "${GITEA_TMP}" ]; then +if [ -z "${GITEA_WORKING_DIR}" ]; then # run gitea backup (user mktemp?) - GITEA_TMP="/tmp/gitea/"; + GITEA_WORKING_DIR="/var/tmp/gitea/"; +fi; +# general temp folder for temporary data storage, this is not working output folder +if [ -z "${GITEA_TEMP_DIR}"]; then + GITEA_TEMP_DIR="/var/tmp"; fi; if [ -z "${GITEA_BIN}" ]; then GITEA_BIN="/usr/local/bin/gitea"; @@ -39,6 +43,10 @@ fi; if [ -z "${GITEA_CONFIG}" ]; then GITEA_CONFIG="/etc/gitea/app.ini" fi; +# This one is not advertised in the config file as it is not recommended to change +if [ -z "${GITEA_EXPORT_TYPE}" ]; then + GITEA_EXPORT_TYPE="zip"; +fi; if [ ! -f "${GITEA_BIN}" ]; then echo "[! $(date +'%F %T')] Cannot find gitea binary"; . "${DIR}/borg.backup.functions.close.sh" 1; @@ -49,6 +57,16 @@ if [ ! -f "${GITEA_CONFIG}" ]; then . "${DIR}/borg.backup.functions.close.sh" 1; exit 1; fi; +# some basic checks with abort +if [ ! -d "${GITEA_TEMP_DIR}" ]; then + echo "Temp directory does not exist: ${GITEA_TEMP_DIR}"; + exit; +fi; +# we should check GITEA_EXPORT_TYPE too at some point for an allow list +# At the moment warn if not zip +if [ "${GITEA_EXPORT_TYPE}" != "zip" ]; then + echo "[!!!!] The gitea export type has been changed from 'zip' to '${GITEA_EXPORT_TYPE}'. This can either break or make exports take very ling"; +fi; # Filename FILENAME="gitea.backup.zip"; # backup set and prefix @@ -60,7 +78,7 @@ BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); printf "${PRINTF_SUB_BLOCK}" "BACKUP: git data and database" "$(date +'%F %T')" "${MODULE}"; if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then - echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}"; + echo "sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_WORKING_DIR} -t ${GITEA_TEMP_DIR} --type ${GITEA_EXPORT_TYPE} -L -f - | ${BORG_CALL}"; if [ -z "${ONE_TIME_TAG}" ]; then echo "${BORG_PRUNE}"; fi; @@ -70,13 +88,13 @@ if [ ${DRYRUN} -eq 0 ]; then # below was an old workaround #export USER="${LOGNAME}" # workaround for broken gitea EUID check # make sure temp folder is there and is set as git. user - if [ ! -d "${GITEA_TMP}" ]; then - mkdir -p "${GITEA_TMP}"; + if [ ! -d "${GITEA_WORKING_DIR}" ]; then + mkdir -p "${GITEA_WORKING_DIR}"; fi; - chown -R ${GIT_USER}. "${GITEA_TMP}"; + chown -R ${GIT_USER}. "${GITEA_WORKING_DIR}"; # this needs to be run in a folder that can be stat by git user - cd "${GITEA_TMP}"; - sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_TMP} -L -f - | ${BORG_CALL}; + cd "${GITEA_WORKING_DIR}"; + sudo -u ${GIT_USER} ${GITEA_BIN} dump -c ${GITEA_CONFIG} -w ${GITEA_WORKING_DIR} -t ${GITEA_TEMP_DIR} --type ${GITEA_EXPORT_TYPE} -L -f - | ${BORG_CALL}; ) 2>&1 | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' # remove all ESC strings fi; if [ -z "${ONE_TIME_TAG}" ]; then diff --git a/borg.backup.mysql.sh b/borg.backup.mysql.sh index 7ab272b..61891ec 100755 --- a/borg.backup.mysql.sh +++ b/borg.backup.mysql.sh @@ -93,6 +93,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='--no-data'; schema_flag='schema'; fi; + LOCAL_START=$(date +'%s'); printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}"; # We only do a full backup and not per table backup here # Filename @@ -124,9 +125,12 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; + DURATION=$[ $(date +'%s')-$LOCAL_START ]; + printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "databases" "${MODULE}" "$(convert_time ${DURATION})"; else ${MYSQL_CMD} ${MYSQL_DB_CONFIG_PARAM} -B -N -e "show databases" | while read db; do + LOCAL_START=$(date +'%s'); printf "${PRINTF_DB_SUB_BLOCK}" "DB" "${db}" "${MODULE}"; printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}"; # exclude checks @@ -213,6 +217,8 @@ else else echo "- [E] ${db}"; fi; + DURATION=$[ $(date +'%s')-$LOCAL_START ]; + printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})"; done; fi; # run compact at the end if not a dry run diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index deb5480..1fa4524 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -102,6 +102,7 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then SCHEMA_ONLY='-s'; schema_flag='schema'; fi; + LOCAL_START=$(date +'%s'); printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "all databases" "$(date +'%F %T')" "${MODULE}"; # Filename FILENAME-"all.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" @@ -132,10 +133,13 @@ if [ ! -z "${DATABASE_FULL_DUMP}" ]; then echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; + DURATION=$[ $(date +'%s')-$LOCAL_START ]; + printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "databases" "${MODULE}" "$(convert_time ${DURATION})"; else # dump globals first db="pg_globals"; schema_flag="data"; + LOCAL_START=$(date +'%s'); printf "${PRINTF_SUBEXT_BLOCK}" "BACKUP" "${db}" "$(date +'%F %T')" "${MODULE}"; # Filename FILENAME="${db}.${DB_USER}.NONE.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" @@ -166,9 +170,11 @@ else echo "Prune repository with keep${KEEP_INFO:1}"; ${BORG_PRUNE}; fi; + printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "BACKUP" "${db}" "${MODULE}" "$(convert_time ${DURATION})"; # get list of tables for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do + LOCAL_START=$(date +'%s'); # get the user who owns the DB too owner=$(echo ${owner_db} | cut -d "," -f 1); db=$(echo ${owner_db} | cut -d "," -f 2); @@ -265,6 +271,7 @@ else else echo "- [E] ${db}"; fi; + printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})"; done; fi; # run compact at the end if not a dry run