From 1b52af909a3db95b97e28b0af1814fa7d24df629 Mon Sep 17 00:00:00 2001 From: Clemens Schwaighofer Date: Thu, 4 Dec 2025 09:28:57 +0900 Subject: [PATCH] Update PostgreSQL module to run as normal user with sudo for PostgreSQL also clean up all PostgreSQL command calls as array type calls clean up binary finder for postgresql installations and add port/host override options --- borg.backup.functions.init.sh | 8 +- borg.backup.pgsql.settings-default | 12 +- borg.backup.pgsql.sh | 193 ++++++++++++++++++++--------- 3 files changed, 155 insertions(+), 58 deletions(-) diff --git a/borg.backup.functions.init.sh b/borg.backup.functions.init.sh index cf2fdae..2d476c6 100644 --- a/borg.backup.functions.init.sh +++ b/borg.backup.functions.init.sh @@ -32,7 +32,7 @@ function version { } # version for all general files -VERSION="4.7.2"; +VERSION="4.7.3"; # borg version and borg comamnd BORG_VERSION=""; @@ -93,6 +93,8 @@ REGEX=""; REGEX_COMMENT="^[\ \t]*#"; REGEX_GLOB='\*'; REGEX_NUMERIC="^[0-9]{1,2}$"; +# port regex, but only approximately +REGEX_PORT="^[0-9]{2,5}$"; REGEX_ERROR="^Some part of the script failed with ERROR:"; PRUNE_DEBUG=""; INIT_REPOSITORY=0; @@ -156,6 +158,10 @@ SUB_BACKUP_SET=""; # for database backup only DATABASE_FULL_DUMP=""; DATABASE_USER=""; +DATABASE_USE_SUDO=""; +DATABASE_SUDO_USER=""; +DATABASE_PORT=""; +DATABASE_HOST=""; # only for mysql old config file MYSQL_DB_CONFIG=""; MYSQL_DB_CONFIG_PARAM=""; diff --git a/borg.backup.pgsql.settings-default b/borg.backup.pgsql.settings-default index 6c67720..49de930 100644 --- a/borg.backup.pgsql.settings-default +++ b/borg.backup.pgsql.settings-default @@ -9,5 +9,15 @@ # note that with this databases that have been dropped need to be pruned manually # if 'schema' word is used, only schema data is dumped DATABASE_FULL_DUMP=""; -# override default postgres user +# override default postgres user and sudo user for all postgres commands +# All commands must be run as the postgres admin user DATABASE_USER=""; +# disable sudo usage by setting to "0", default is to use sudo +DATABASE_USE_SUDO=""; +# set the sudo user, if not set postgres or DATABASE_USER is used +DATABASE_SUDO_USER=""; +# override port +DATABASE_PORT=""; +# override database host, if set to local, localhost or 127.0.0.1 it will use sockets to connect +# for socket connection ident or sudo has to be used +DATABASE_HOST=""; diff --git a/borg.backup.pgsql.sh b/borg.backup.pgsql.sh index c28d03f..7e9eb08 100755 --- a/borg.backup.pgsql.sh +++ b/borg.backup.pgsql.sh @@ -10,11 +10,13 @@ # set last edit date + time MODULE="pgsql" -MODULE_VERSION="1.2.8"; +MODULE_VERSION="1.3.0"; DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +DEFAULT_DATABASE_USE_SUDO="1"; +DEFAULT_DATABASE_USER="postgres"; # init system . "${DIR}/borg.backup.functions.init.sh"; @@ -35,15 +37,41 @@ BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock"; # if info print info and then abort run . "${DIR}/borg.backup.functions.info.sh"; +# set db and sudo user if [ -n "${DATABASE_USER}" ]; then - DB_USER=${DATABASE_USER}; + DB_USER="${DATABASE_USER}"; else - DB_USER='postgres'; + DB_USER="${DEFAULT_DATABASE_USER}"; fi; +if [ -n "${DATABASE_SUDO_USER}" ]; then + DB_SUDO_USER="${DATABASE_SUDO_USER}"; +else + DB_SUDO_USER="${DB_USER}"; +fi; +# set flag if we should use sudo +if [ -n "${DATABASE_USE_SUDO}" ]; then + USE_SUDO="${DATABASE_USE_SUDO}"; +else + USE_SUDO="${DEFAULT_DATABASE_USE_SUDO}"; +fi; +# sudo command, or none if not requested +SUDO_COMMAND_A=("sudo" "-u" "${DB_SUDO_USER}"); +if [ "${USE_SUDO}" -eq "0" ]; then + SUDO_COMMAND_A=() +fi; + # get current pgsql version first # if first part <10 then user full, else only first part # eg 9.4 -> 9.4, 12.5 -> 12 -PG_VERSION=$(pgv=$(psql -U ${DB_USER} -d template1 -t -A -F "," -X -q -c 'select version();' | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/'); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi ); +PG_VERSION_CMD=("${SUDO_COMMAND_A[@]}" "psql" "-U" "${DB_USER}" "-d" "template1" "-t" "-A" "-F" "," "-X" "-q" "-c" "select version();"); +PG_VERSION=$( + pgv=$("${PG_VERSION_CMD[@]}" | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/'); + if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then + echo "${pgv}" | cut -d "." -f 1; + else + echo "${pgv}" | cut -d "." -f 1,2; + fi; +); _PATH_PG_VERSION=${PG_VERSION}; _backup_error=$?; if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then @@ -53,49 +81,75 @@ if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then fi; # path set per Distribution type and current running DB version +# Debian/Ubuntu: PG_BASE_PATH='/usr/lib/postgresql/'; # Redhat: PG_BASE_PATH='/usr/pgsql-'; -# AWS 1: PG_BASE_PATH='/usr/lib64/pgsql'; -# Debian: PG_BASE_PATH='/usr/lib/postgresql/'; -PG_BASE_PATH='/usr/lib/postgresql/'; -if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then - PG_BASE_PATH='/usr/pgsql-'; - if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then - PG_BASE_PATH='/usr/lib64/pgsql'; - _PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//'); - if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then - echo "[! $(date +'%F %T')] PostgreSQL not found in any paths"; - . "${DIR}/borg.backup.functions.close.sh" 1; - exit 1; - fi; +PG_BASE_PATH_LIST=("/usr/lib/postgresql" "/usr/lib64/pgsql"); +PG_BASE_PATH=""; +for path in "${PG_BASE_PATH_LIST[@]}"; do + if [ -f "${path}/${_PATH_PG_VERSION}/bin/psql" ]; then + PG_BASE_PATH="${path}/"; + break; fi; +done; +if [ -z "${PG_BASE_PATH}" ]; then + echo "[! $(date +'%F %T')] PostgreSQL not found in any paths"; + . "${DIR}/borg.backup.functions.close.sh" 1; + exit 1; fi; -PG_PATH=${PG_BASE_PATH}${_PATH_PG_VERSION}'/bin/'; -PG_PSQL=${PG_PATH}'psql'; -PG_DUMP=${PG_PATH}'pg_dump'; -PG_DUMPALL=${PG_PATH}'pg_dumpall'; +PG_PATH="${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/"; +PG_PSQL=("${PG_PATH}psql"); +PG_DUMP=("${PG_PATH}pg_dump"); +PG_DUMPALL=("${PG_PATH}pg_dumpall"); +PG_ERROR=0 # check that command are here -if [ ! -f "${PG_PSQL}" ]; then - echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}"; - . "${DIR}/borg.backup.functions.close.sh" 1; - exit 1; +if [ ! -f "${PG_PSQL[0]}" ]; then + echo "[!] ($(date +'%F %T')) psql binary not found in ${PG_PATH}"; + PG_ERROR=1; fi; -if [ ! -f "${PG_DUMP}" ]; then - echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}"; - . "${DIR}/borg.backup.functions.close.sh" 1; - exit 1; +if [ ! -f "${PG_DUMP[0]}" ]; then + echo "[!] ($(date +'%F %T')) pg_dump binary not found in ${PG_PATH}"; + PG_ERROR=1; fi; -if [ ! -f "${PG_DUMPALL}" ]; then - echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}"; +if [ ! -f "${PG_DUMPALL[0]}" ]; then + echo "[!] ($(date +'%F %T')) pg_dumpall binary not found in ${PG_PATH}"; + PG_ERROR=1; +fi; +if [ ${PG_ERROR} -ne 0 ]; then . "${DIR}/borg.backup.functions.close.sh" 1; - exit 1; + exit ${PG_ERROR}; +fi; +# prefix with sudo, if sudo command is requested +if [ "${USE_SUDO}" -ne "0" ]; then + PG_PSQL=("${SUDO_COMMAND_A[@]}" "${PG_PSQL[@]}"); + PG_DUMP=("${SUDO_COMMAND_A[@]}" "${PG_DUMP[@]}"); + PG_DUMPALL=("${SUDO_COMMAND_A[@]}" "${PG_DUMPALL[@]}"); fi; DB_VERSION=${PG_VERSION}; -# TODO override port/host info -DB_PORT='5432'; -DB_HOST='local'; # or -CONN_DB_HOST=''; # -h -CONN_DB_PORT=''; # -p +# override for port or host name +if [ -n "${DATABASE_PORT}" ] && [[ "${DATABASE_PORT}" =~ ${REGEX_PORT} ]]; then + DB_PORT="${DATABASE_PORT}"; +else + DB_PORT="5432"; +fi; +if [ -n "${DATABASE_HOST}" ]; then + DB_HOST="${DATABASE_HOST}"; +else + DB_HOST="local"; +fi; +# use socket for local +if [ "${DB_HOST}" = "local" ] || [ "${DB_HOST}" = "localhost" ] || [ "${DB_HOST}" = "127.0.0.1" ]; then + DB_HOST='local'; + CONN_DB_HOST=(); + CONN_DB_PORT=(); +else + CONN_DB_HOST=("-p" "${DB_HOST}"); # -h + CONN_DB_PORT=("-h" "${DB_HOSTNAME}"); # -p +fi; +# now add user, con, host to all commands +PG_PSQL=("${PG_PSQL[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}"); +PG_DUMP=("${PG_DUMP[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}"); +PG_DUMPALL=("${PG_DUMPALL[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}"); # ALL IN ONE FILE or PER DATABASE FLAG if [ -n "${DATABASE_FULL_DUMP}" ]; then @@ -113,17 +167,25 @@ if [ -n "${DATABASE_FULL_DUMP}" ]; then BACKUP_SET_PREFIX="${MODULE},all-"; BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call - BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); - BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + BORG_CALL=$( + echo "${_BORG_CALL}" | + sed -e "s/##FILENAME##/${FILENAME}/" | + sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/" + ); + BORG_PRUNE=$( + echo "${_BORG_PRUNE}" | + sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/" + ); + PG_DUMPALL_CMD=("${PG_DUMPALL[@]}" "${SCHEMA_ONLY}" "-c"); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; - echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}"; + echo "${PG_DUMPALL_CMD[*]} | ${BORG_CALL}"; if [ -z "${ONE_TIME_TAG}" ]; then echo "${BORG_PRUNE}"; fi; fi; if [ ${DRYRUN} -eq 0 ]; then - ${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}; + "${PG_DUMPALL_CMD[@]}" | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}"; @@ -150,17 +212,25 @@ else BACKUP_SET_PREFIX="${MODULE},${db}-"; BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call - BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); - BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + BORG_CALL=$( + echo "${_BORG_CALL}" | + sed -e "s/##FILENAME##/${FILENAME}/" | + sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/" + ); + BORG_PRUNE=$( + echo "${_BORG_PRUNE}" | + sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/" + ); + PG_DUMPALL_CMD=("${PG_DUMPALL[@]}" "--globals-only"); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; - echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}"; + echo "${PG_DUMPALL_CMD[*]} | ${BORG_CALL}"; if [ -z "${ONE_TIME_TAG}" ]; then echo "${BORG_PRUNE}"; fi; fi; if [ ${DRYRUN} -eq 0 ]; then - ${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}; + "${PG_DUMPALL_CMD[@]}" | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}"; @@ -177,7 +247,11 @@ else printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})"; # get list of tables - for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"); do + GET_DB_CMD=( + "${PG_PSQL[@]}" "-d" "template1" "-t" "-A" "-F" "," "-X" "-q" "-c" + "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;" + ); + for owner_db in $("${GET_DB_CMD[@]}"); do LOCAL_START=$(date +'%s'); # get the user who owns the DB too owner=$(echo "${owner_db}" | cut -d "," -f 1); @@ -206,17 +280,15 @@ else done<"${BASE_FOLDER}${EXCLUDE_FILE}"; fi; if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then + PG_DUMP_CMD=("${PG_DUMP[@]}" "-c" "--format=c"); # set dump type - SCHEMA_ONLY=''; schema_flag=''; # schema or data # schema exclude over data exclude, can't have both if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then # default is data dump - SCHEMA_ONLY=''; schema_flag='data'; while read -r schema_db; do if [ "${db}" = "${schema_db}" ]; then - SCHEMA_ONLY='-s'; schema_flag='schema'; # skip out break; @@ -224,11 +296,9 @@ else done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}"; elif [ -s "${BASE_FOLDER}${DATA_ONLY_FILE}" ]; then # default to schema, unless in data list - SCHEMA_ONLY='-s'; schema_flag='schema'; while read -r data_db; do if [ "${db}" = "${data_db}" ]; then - SCHEMA_ONLY=''; schema_flag='data'; # skip out break; @@ -237,9 +307,13 @@ else fi; # if nothing is set, default to data if [ -z "${schema_flag}" ]; then - SCHEMA_ONLY='' schema_flag="data"; fi; + # set schema flag if schmea only is requested + if [ $schema_flag = "schema" ]; then + PG_DUMP_CMD+=("-s"); + fi; + PG_DUMP_CMD+=("${db}"); # Filename # Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql" @@ -248,18 +322,25 @@ else # backup set: BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}"; # borg call - BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"); + BORG_CALL=$( + echo "${_BORG_CALL}" | + sed -e "s/##FILENAME##/${FILENAME}/" | + sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/" + ); # borg prune - BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"); + BORG_PRUNE=$( + echo "${_BORG_PRUNE}" | + sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/" + ); if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";"; - echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}"; + echo "${PG_DUMP_CMD[*]} | ${BORG_CALL}"; if [ -z "${ONE_TIME_TAG}" ]; then echo "${BORG_PRUNE}"; fi; fi; if [ ${DRYRUN} -eq 0 ]; then - ${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c "${db}" | ${BORG_CALL}; + "${PG_DUMP_CMD[@]}" | ${BORG_CALL}; _backup_error=$?; if [ $_backup_error -ne 0 ]; then echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";