Update PostgreSQL module to run as normal user with sudo for PostgreSQL

also clean up all PostgreSQL command calls as array type calls
clean up binary finder for postgresql installations
and add port/host override options

borg.backup.functions.init.sh

@@ -32,7 +32,7 @@ function version {
 }
 
 # version for all general files
-VERSION="4.7.2";
+VERSION="4.7.3";
 
 # borg version and borg comamnd
 BORG_VERSION="";
@@ -93,6 +93,8 @@ REGEX="";
 REGEX_COMMENT="^[\ \t]*#";
 REGEX_GLOB='\*';
 REGEX_NUMERIC="^[0-9]{1,2}$";
+# port regex, but only approximately
+REGEX_PORT="^[0-9]{2,5}$";
 REGEX_ERROR="^Some part of the script failed with ERROR:";
 PRUNE_DEBUG="";
 INIT_REPOSITORY=0;
@@ -156,6 +158,10 @@ SUB_BACKUP_SET="";
 # for database backup only
 DATABASE_FULL_DUMP="";
 DATABASE_USER="";
+DATABASE_USE_SUDO="";
+DATABASE_SUDO_USER="";
+DATABASE_PORT="";
+DATABASE_HOST="";
 # only for mysql old config file
 MYSQL_DB_CONFIG="";
 MYSQL_DB_CONFIG_PARAM="";

borg.backup.pgsql.settings-default

@@ -9,5 +9,15 @@
 # note that with this databases that have been dropped need to be pruned manually
 # if 'schema' word is used, only schema data is dumped
 DATABASE_FULL_DUMP="";
-# override default postgres user
+# override default postgres user and sudo user for all postgres commands
+# All commands must be run as the postgres admin user
 DATABASE_USER="";
+# disable sudo usage by setting to "0", default is to use sudo
+DATABASE_USE_SUDO="";
+# set the sudo user, if not set postgres or DATABASE_USER is used
+DATABASE_SUDO_USER="";
+# override port
+DATABASE_PORT="";
+# override database host, if set to local, localhost or 127.0.0.1 it will use sockets to connect
+# for socket connection ident or sudo has to be used
+DATABASE_HOST="";

borg.backup.pgsql.sh

@@ -10,11 +10,13 @@
 
 # set last edit date + time
 MODULE="pgsql"
-MODULE_VERSION="1.2.8";
+MODULE_VERSION="1.3.0";
 
 
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+DEFAULT_DATABASE_USE_SUDO="1";
+DEFAULT_DATABASE_USER="postgres";
 # init system
 . "${DIR}/borg.backup.functions.init.sh";
 
@@ -35,15 +37,41 @@ BACKUP_LOCK_FILE="borg.backup.${MODULE}.lock";
 # if info print info and then abort run
 . "${DIR}/borg.backup.functions.info.sh";
 
+# set db and sudo user
 if [ -n "${DATABASE_USER}" ]; then
-	DB_USER=${DATABASE_USER};
+	DB_USER="${DATABASE_USER}";
 else
-	DB_USER='postgres';
+	DB_USER="${DEFAULT_DATABASE_USER}";
 fi;
+if [ -n "${DATABASE_SUDO_USER}" ]; then
+	DB_SUDO_USER="${DATABASE_SUDO_USER}";
+else
+	DB_SUDO_USER="${DB_USER}";
+fi;
+# set flag if we should use sudo
+if [ -n "${DATABASE_USE_SUDO}" ]; then
+	USE_SUDO="${DATABASE_USE_SUDO}";
+else
+	USE_SUDO="${DEFAULT_DATABASE_USE_SUDO}";
+fi;
+# sudo command, or none if not requested
+SUDO_COMMAND_A=("sudo" "-u" "${DB_SUDO_USER}");
+if [ "${USE_SUDO}" -eq "0" ]; then
+	SUDO_COMMAND_A=()
+fi;
+
 # get current pgsql version first
 # if first part <10 then user full, else only first part
 # eg 9.4 -> 9.4, 12.5 -> 12
-PG_VERSION=$(pgv=$(psql -U ${DB_USER} -d template1 -t -A -F "," -X -q -c 'select version();' | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/'); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi );
+PG_VERSION_CMD=("${SUDO_COMMAND_A[@]}" "psql" "-U" "${DB_USER}" "-d" "template1" "-t" "-A" "-F" "," "-X" "-q" "-c" "select version();");
+PG_VERSION=$(
+	pgv=$("${PG_VERSION_CMD[@]}" | sed -e 's/^PostgreSQL \([0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/');
+	if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then
+		echo "${pgv}" | cut -d "." -f 1;
+	else
+		echo "${pgv}" | cut -d "." -f 1,2;
+	fi;
+);
 _PATH_PG_VERSION=${PG_VERSION};
 _backup_error=$?;
 if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then
@@ -53,49 +81,75 @@ if [ $_backup_error -ne 0 ] || [ -z "${PG_VERSION}" ]; then
 fi;
 
 # path set per Distribution type and current running DB version
+# Debian/Ubuntu: PG_BASE_PATH='/usr/lib/postgresql/';
 # Redhat: PG_BASE_PATH='/usr/pgsql-';
-# AWS 1: PG_BASE_PATH='/usr/lib64/pgsql';
+PG_BASE_PATH_LIST=("/usr/lib/postgresql" "/usr/lib64/pgsql");
-# Debian: PG_BASE_PATH='/usr/lib/postgresql/';
+PG_BASE_PATH="";
-PG_BASE_PATH='/usr/lib/postgresql/';
+for path in "${PG_BASE_PATH_LIST[@]}"; do
-if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
+	if [ -f "${path}/${_PATH_PG_VERSION}/bin/psql" ]; then
-	PG_BASE_PATH='/usr/pgsql-';
+		PG_BASE_PATH="${path}/";
-	if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
+		break;
-		PG_BASE_PATH='/usr/lib64/pgsql';
+	fi;
-		_PATH_PG_VERSION=$(echo "${PG_VERSION}" | sed -e 's/\.//');
+done;
-		if [ ! -f "${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/psql" ]; then
+if [ -z "${PG_BASE_PATH}" ]; then
 	echo "[! $(date +'%F %T')] PostgreSQL not found in any paths";
 		. "${DIR}/borg.backup.functions.close.sh" 1;
 	exit 1;
 fi;
-	fi;
+PG_PATH="${PG_BASE_PATH}${_PATH_PG_VERSION}/bin/";
-fi;
+PG_PSQL=("${PG_PATH}psql");
-PG_PATH=${PG_BASE_PATH}${_PATH_PG_VERSION}'/bin/';
+PG_DUMP=("${PG_PATH}pg_dump");
-PG_PSQL=${PG_PATH}'psql';
+PG_DUMPALL=("${PG_PATH}pg_dumpall");
-PG_DUMP=${PG_PATH}'pg_dump';
+PG_ERROR=0
-PG_DUMPALL=${PG_PATH}'pg_dumpall';
 # check that command are here
-if [ ! -f "${PG_PSQL}" ]; then
+if [ ! -f "${PG_PSQL[0]}" ]; then
-	echo "[! $(date +'%F %T')] psql binary not found in ${PG_PATH}";
+	echo "[!] ($(date +'%F %T')) psql binary not found in ${PG_PATH}";
-	. "${DIR}/borg.backup.functions.close.sh" 1;
+	PG_ERROR=1;
-	exit 1;
 fi;
-if [ ! -f "${PG_DUMP}" ]; then
+if [ ! -f "${PG_DUMP[0]}" ]; then
-	echo "[! $(date +'%F %T')] pg_dump binary not found in ${PG_PATH}";
+	echo "[!] ($(date +'%F %T')) pg_dump binary not found in ${PG_PATH}";
-	. "${DIR}/borg.backup.functions.close.sh" 1;
+	PG_ERROR=1;
-	exit 1;
 fi;
-if [ ! -f "${PG_DUMPALL}" ]; then
+if [ ! -f "${PG_DUMPALL[0]}" ]; then
-	echo "[! $(date +'%F %T')] pg_dumpall binary not found in ${PG_PATH}";
+	echo "[!] ($(date +'%F %T')) pg_dumpall binary not found in ${PG_PATH}";
+	PG_ERROR=1;
+fi;
+if [ ${PG_ERROR} -ne 0 ]; then
 	. "${DIR}/borg.backup.functions.close.sh" 1;
-	exit 1;
+	exit ${PG_ERROR};
+fi;
+# prefix with sudo, if sudo command is requested
+if [ "${USE_SUDO}" -ne "0" ]; then
+	PG_PSQL=("${SUDO_COMMAND_A[@]}" "${PG_PSQL[@]}");
+	PG_DUMP=("${SUDO_COMMAND_A[@]}" "${PG_DUMP[@]}");
+	PG_DUMPALL=("${SUDO_COMMAND_A[@]}" "${PG_DUMPALL[@]}");
 fi;
 
 DB_VERSION=${PG_VERSION};
-# TODO override port/host info
+# override for port or host name
-DB_PORT='5432';
+if [ -n "${DATABASE_PORT}" ] && [[ "${DATABASE_PORT}" =~ ${REGEX_PORT} ]]; then
-DB_HOST='local'; # or <host>
+	DB_PORT="${DATABASE_PORT}";
-CONN_DB_HOST=''; # -h <host>
+else
-CONN_DB_PORT=''; # -p <port>
+	DB_PORT="5432";
+fi;
+if [ -n "${DATABASE_HOST}" ]; then
+	DB_HOST="${DATABASE_HOST}";
+else
+	DB_HOST="local";
+fi;
+# use socket for local
+if [ "${DB_HOST}" = "local" ] || [ "${DB_HOST}" = "localhost" ] || [ "${DB_HOST}" = "127.0.0.1" ]; then
+	DB_HOST='local';
+	CONN_DB_HOST=();
+	CONN_DB_PORT=();
+else
+	CONN_DB_HOST=("-p" "${DB_HOST}"); # -h <host>
+	CONN_DB_PORT=("-h" "${DB_HOSTNAME}"); # -p <port>
+fi;
+# now add user, con, host to all commands
+PG_PSQL=("${PG_PSQL[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}");
+PG_DUMP=("${PG_DUMP[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}");
+PG_DUMPALL=("${PG_DUMPALL[@]}" "-U" "${DB_USER}" "${CONN_DB_HOST[@]}" "${CONN_DB_PORT[@]}");
 
 # ALL IN ONE FILE or PER DATABASE FLAG
 if [ -n "${DATABASE_FULL_DUMP}" ]; then
@@ -113,17 +167,25 @@ if [ -n "${DATABASE_FULL_DUMP}" ]; then
 	BACKUP_SET_PREFIX="${MODULE},all-";
 	BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 	# borg call
-	BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
+	BORG_CALL=$(
-	BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
+		echo "${_BORG_CALL}" |
+		sed -e "s/##FILENAME##/${FILENAME}/" |
+		sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"
+	);
+	BORG_PRUNE=$(
+		echo "${_BORG_PRUNE}" |
+		sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"
+	);
+	PG_DUMPALL_CMD=("${PG_DUMPALL[@]}" "${SCHEMA_ONLY}" "-c");
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
-		echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL}";
+		echo "${PG_DUMPALL_CMD[*]} | ${BORG_CALL}";
 		if [ -z "${ONE_TIME_TAG}" ]; then
 			echo "${BORG_PRUNE}";
 		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
-		${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} ${SCHEMA_ONLY} -c | ${BORG_CALL};
+		"${PG_DUMPALL_CMD[@]}" | ${BORG_CALL};
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for full dump with error code: ${_backup_error}";
@@ -150,17 +212,25 @@ else
 	BACKUP_SET_PREFIX="${MODULE},${db}-";
 	BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 	# borg call
-	BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
+	BORG_CALL=$(
-	BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
+		echo "${_BORG_CALL}" |
+		sed -e "s/##FILENAME##/${FILENAME}/" |
+		sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"
+	);
+	BORG_PRUNE=$(
+		echo "${_BORG_PRUNE}" |
+		sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"
+	);
+	PG_DUMPALL_CMD=("${PG_DUMPALL[@]}" "--globals-only");
 	if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 		echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
-		echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL}";
+		echo "${PG_DUMPALL_CMD[*]} | ${BORG_CALL}";
 		if [ -z "${ONE_TIME_TAG}" ]; then
 			echo "${BORG_PRUNE}";
 		fi;
 	fi;
 	if [ ${DRYRUN} -eq 0 ]; then
-		${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} --globals-only | ${BORG_CALL};
+		"${PG_DUMPALL_CMD[@]}" | ${BORG_CALL};
 		_backup_error=$?;
 		if [ $_backup_error -ne 0 ]; then
 			echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";
@@ -177,7 +247,11 @@ else
 	printf "${PRINTF_DB_RUN_TIME_SUB_BLOCK}" "DONE" "${db}" "${MODULE}" "$(convert_time ${DURATION})";
 
 	# get list of tables
-	for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"); do
+	GET_DB_CMD=(
+		"${PG_PSQL[@]}" "-d" "template1" "-t" "-A" "-F" "," "-X" "-q" "-c"
+		"SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"
+	);
+	for owner_db in $("${GET_DB_CMD[@]}"); do
 		LOCAL_START=$(date +'%s');
 		# get the user who owns the DB too
 		owner=$(echo "${owner_db}" | cut -d "," -f 1);
@@ -206,17 +280,15 @@ else
 			done<"${BASE_FOLDER}${EXCLUDE_FILE}";
 		fi;
 		if [ ${include} -eq 1 ] && [ ${exclude} -eq 0 ]; then
+			PG_DUMP_CMD=("${PG_DUMP[@]}" "-c" "--format=c");
 			# set dump type
-			SCHEMA_ONLY='';
 			schema_flag=''; # schema or data
 			# schema exclude over data exclude, can't have both
 			if [ -s "${BASE_FOLDER}${SCHEMA_ONLY_FILE}" ]; then
 				# default is data dump
-				SCHEMA_ONLY='';
 				schema_flag='data';
 				while read -r schema_db; do
 					if [ "${db}" = "${schema_db}" ]; then
-						SCHEMA_ONLY='-s';
 						schema_flag='schema';
 						# skip out
 						break;
@@ -224,11 +296,9 @@ else
 				done<"${BASE_FOLDER}${SCHEMA_ONLY_FILE}";
 			elif [ -s "${BASE_FOLDER}${DATA_ONLY_FILE}" ]; then
 				# default to schema, unless in data list
-				SCHEMA_ONLY='-s';
 				schema_flag='schema';
 				while read -r data_db; do
 					if [ "${db}" = "${data_db}" ]; then
-						SCHEMA_ONLY='';
 						schema_flag='data';
 						# skip out
 						break;
@@ -237,9 +307,13 @@ else
 			fi;
 			# if nothing is set, default to data
 			if [ -z "${schema_flag}" ]; then
-				SCHEMA_ONLY=''
 				schema_flag="data";
 			fi;
+			# set schema flag if schmea only is requested
+			if [ $schema_flag = "schema" ]; then
+				PG_DUMP_CMD+=("-s");
+			fi;
+			PG_DUMP_CMD+=("${db}");
 			# Filename
 			# Database.User.Encoding.pgsql|data|schema-Version_Host_Port_YearMonthDay_HourMinute_Counter.Fromat(c).sql
 			FILENAME="${db}.${owner}.${encoding}.${schema_flag}-${DB_VERSION}_${DB_HOST}_${DB_PORT}.c.sql"
@@ -248,18 +322,25 @@ else
 			# backup set:
 			BACKUP_SET_NAME="${ONE_TIME_TAG}${BACKUP_SET_PREFIX}${schema_flag}-${BACKUP_SET}";
 			# borg call
-			BORG_CALL=$(echo "${_BORG_CALL}" | sed -e "s/##FILENAME##/${FILENAME}/" | sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/");
+			BORG_CALL=$(
+				echo "${_BORG_CALL}" |
+				sed -e "s/##FILENAME##/${FILENAME}/" |
+				sed -e "s/##BACKUP_SET##/${BACKUP_SET_NAME}/"
+			);
 			# borg prune
-			BORG_PRUNE=$(echo "${_BORG_PRUNE}" | sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/");
+			BORG_PRUNE=$(
+				echo "${_BORG_PRUNE}" |
+				sed -e "s/##BACKUP_SET_PREFIX##/${BACKUP_SET_PREFIX}/"
+			);
 			if [ ${DEBUG} -eq 1 ] || [ ${DRYRUN} -eq 1 ]; then
 				echo "export BORG_BASE_DIR=\"${BASE_FOLDER}\";";
-				echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c ${db} | ${BORG_CALL}";
+				echo "${PG_DUMP_CMD[*]} | ${BORG_CALL}";
 				if [ -z "${ONE_TIME_TAG}" ]; then
 					echo "${BORG_PRUNE}";
 				fi;
 			fi;
 			if [ ${DRYRUN} -eq 0 ]; then
-				${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} ${CONN_DB_PORT} -c ${SCHEMA_ONLY} --format=c "${db}" | ${BORG_CALL};
+				"${PG_DUMP_CMD[@]}" | ${BORG_CALL};
 				_backup_error=$?;
 				if [ $_backup_error -ne 0 ]; then
 					echo "[! $(date +'%F %T')] Backup creation failed for ${db} dump with error code: ${_backup_error}";