Add -g (go) option, add force new key creation, test script add

Remove and Rotate script have -g (go) flag for actual run.
Remove has text updates for removed key info
Rotate force only forces rotation, but will not create new key unless -c (force create) is set
Test script added for testing connections

bin/remove-old-ssh-keys.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# Remove previous keys
+
 # base folder for all data
 BASE_FOLDER=$(dirname "$(readlink -f "$0")")"/";
 # config folder
@@ -11,10 +13,11 @@ ADMIN_USERS=(admin ubuntu ec2-user)
 
 DRY_RUN=0;
 FORCE=0;
+GO=0;
 HOST_ONLY="";
 USER_ONLY="";
 #
-while getopts ":h:u:nf" opt; do
+while getopts ":h:u:nfg" opt; do
 	case "${opt}" in
 		h|hostname)
 			HOST_ONLY="${OPTARG}";
@@ -28,12 +31,16 @@ while getopts ":h:u:nf" opt; do
 		f|force)
 			FORCE=1;
 			;;
+		g|go)
+			GO=1;
+			;;
 		\?)
 			echo -e "\n Option does not exist: ${OPTARG}\n";
 			echo "-h override single host name";
 			echo "-u override user name for a host";
 			echo "-f force key change";
 			echo "-n dry run";
+			echo "-g flag for actual change call";
 			echo ""
 			exit 1;
 			;;
@@ -54,8 +61,16 @@ if [ ! -f "${CONFIG_BASE}${server_list}" ]; then
 	echo "Cannot find ${server_list} file in the config folder";
 	exit
 fi
+# abort if go not set
+if [ ${GO} -eq 0 ] && [ ${DRY_RUN} -eq 1 ]; then
+	GO=1;
+elif [ ${GO} -eq 0 ]; then
+	echo "No -g (go) parameter set. aborting. For testing set -n for dry run"
+	exit;
+fi
 
 # default ssh command
+# -t is needed for systens when "Defaults    requiretty" is set
 SSH="ssh -a -x";
 
 remove_ssh_key() {
@@ -121,7 +136,8 @@ for line in `cat "${CONFIG_BASE}${server_list}" | sed 1d`; do
 
 	# previous public key does not exist, skip
 	if [ ! -f "${SSH_PUBLIC_KEYS_PREVIOUS}${SSH_KEY_PUB_FILE}" ]; then
-		echo "[!] Missing previous public key file ${SSH_KEY_PUB_FILE} for ${username}@${hostname}";
+		echo "[!] No previous public key file ${SSH_KEY_PUB_FILE} for ${username}@${hostname}";
+		echo "[_] ............... SKIP";
 		continue;
 	fi
 

bin/rotate-ssh-keys.sh

@@ -2,12 +2,6 @@
 
 # Rotate and deploy admin keys
 
-# List of servers with info
-
-# ssh-keygen -t ed25519 -N "" -C "<COMMENT>" -f <FILE>
-
-# store old key names, current key names
-
 # base folder for all data
 BASE_FOLDER=$(dirname "$(readlink -f "$0")")"/";
 # config folder
@@ -24,10 +18,12 @@ ADMIN_USERS=(admin ubuntu ec2-user)
 
 DRY_RUN=0;
 FORCE=0;
+FORCE_CREATE=0;
+GO=0;
 HOST_ONLY="";
 USER_ONLY="";
 #
-while getopts ":h:u:nf" opt; do
+while getopts ":h:u:nfg" opt; do
 	case "${opt}" in
 		h|hostname)
 			HOST_ONLY="${OPTARG}";
@@ -41,12 +37,20 @@ while getopts ":h:u:nf" opt; do
 		f|force)
 			FORCE=1;
 			;;
+		c|force-create)
+			FORCE_CREATE=1
+			;;
+		g|go)
+			GO=1;
+			;;
 		\?)
 			echo -e "\n Option does not exist: ${OPTARG}\n";
 			echo "-h override single host name";
 			echo "-u override user name for a host";
 			echo "-f force key change";
+			echo "-c force create new key even if old key exists";
 			echo "-n dry run";
+			echo "-g flag for actual change call";
 			echo ""
 			exit 1;
 			;;
@@ -103,8 +107,16 @@ if [ ! -d "${PEM_ARCHIVE}" ]; then
 fi
 # add todays date
 PEM_ARCHIVE="${PEM_ARCHIVE}/$(date +%F)/"
+# abort if go not set
+if [ ${GO} -eq 0 ] && [ ${DRY_RUN} -eq 1 ]; then
+	GO=1;
+elif [ ${GO} -eq 0 ]; then
+	echo "No -g (go) parameter set. aborting. For testing set -n for dry run"
+	exit;
+fi
 
 # default ssh command
+# -t is needed for systens when "Defaults    requiretty" is set
 SSH="ssh -a -x";
 
 # Add the SSH Key to an auth file if it does not exist yet and the auth file does exist
@@ -203,7 +215,7 @@ for line in `cat "${CONFIG_BASE}${server_list}" | sed 1d`; do
 	# else create new
 	CREATE_NEW_KEY=0;
 	# if we have force, override this all
-	if [ ${FORCE} -eq 1 ]; then
+	if [ ${FORCE_CREATE} -eq 1 ]; then
 		CREATE_NEW_KEY=1;
 	elif [ -f "${SSH_PRIVATE_KEYS}${SSH_KEY_FILE}" ] || [ -f "${SSH_PUBLIC_KEYS_CURRENT}${SSH_KEY_PUB_FILE}" ]; then
 		# if we miss private key -> alert skip
@@ -304,5 +316,4 @@ for line in `cat "${CONFIG_BASE}${server_list}" | sed 1d`; do
 	echo "[=] ............... DONE";
 done
 
-
 # __END__

bin/test-ssh-keys.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+# Test connections
+
+BASE_FOLDER=$(dirname "$(readlink -f "$0")")"/";
+# config folder
+CONFIG_BASE="${BASE_FOLDER}../config/";
+# timestamps of last rotate per user/host
+LAST_ROTATE="${BASE_FOLDER}../last-rotate/";
+
+HOST_ONLY="";
+USER_ONLY="";
+while getopts ":h:u:nfg" opt; do
+	case "${opt}" in
+		h|hostname)
+			HOST_ONLY="${OPTARG}";
+			;;
+		u|username)
+			USER_ONLY="${OPTARG}";
+			;;
+		\?)
+			echo -e "\n Option does not exist: ${OPTARG}\n";
+			echo "-h override single host name";
+			echo "-u override user name for a host";
+			echo ""
+			exit 1;
+			;;
+	esac
+done
+
+# load config
+if [ -f "${CONFIG_BASE}settings.ini" ]; then
+	source <(grep = "${CONFIG_BASE}settings.ini" | sed 's/ *= */=/g')
+fi
+# we must have "server_list" set and file must be in config folder
+if [ ! -f "${CONFIG_BASE}${server_list}" ]; then
+	echo "Cannot find ${server_list} file in the config folder";
+	exit;
+fi
+
+SSH="ssh -a -x";
+# date command
+datecmd="date";
+if date --version 2>&1 | grep -qi "date: illegal option"; then
+	# if this is macOS date, see if gdate exists and use this one
+	type gdate >/dev/null || error "macOS date does not support -d and no GNU date installed";
+	datecmd="gdate";
+fi
+TEST_STRING="TEST";
+ERROR=0;
+WARNING=0;
+
+for line in `cat "${CONFIG_BASE}${server_list}" | sed 1d`; do
+	if [[ "${i}" =~ ^\# ]]; then
+		continue;
+	fi
+	# hostname is on pos 1
+	hostname=$(echo "${line}" | cut -d "," -f 1);
+	# if hostname opt set and not matching skip
+	if [ ! -z "${HOST_ONLY}" ] && [ "${HOST_ONLY}" != "${hostname}" ]; then
+		continue;
+	fi
+	# login user name
+	username=$(echo "${line}" | cut -d "," -f 2);
+	# if username opt set and not matching skip
+	if [ ! -z "${USER_ONLY}" ] && [ "${USER_ONLY}" != "${username}" ]; then
+		continue;
+	fi
+
+	echo -n "[:] Test connection to: ${username}@${hostname}: ";
+	response=$(${SSH} -S none "${username}"@"${hostname}" echo "${TEST_STRING}");
+	if [ "${response}" = "${TEST_STRING}" ]; then
+		# get the SSH key with "${username}"@"${hostname}": YYYY-MM-DD to confirm the correct one is set
+		current_ssh_key=$(
+			${SSH} -S none "${username}"@"${hostname}" grep "${username}@${hostname}: " /etc/ssh/authorized_keys--master | cut -d " " -f 3,4
+		);
+		# extract username@hostname and create date
+		current_user_hostname=$(echo "${current_ssh_key}" | cut -d ":" -f 1);
+		current_date=$(echo "${current_ssh_key}" | cut -d ":" -f 2);
+		current_date=${current_date//[[:blank:]]/};
+		rotate_date="";
+		if [ -f "${LAST_ROTATE}${hostname}_${username}.last-rotate" ]; then
+			# load last rotate timestamp and get YYYY-MM-DD
+			rotate_date=$(${datecmd} -d @$(cat "${LAST_ROTATE}${hostname}_${username}.last-rotate") +%F);
+		fi
+		# check for diff in username / hostname
+		warning_string=""
+		if [ "${current_user_hostname}" != "${username}@${hostname}" ]; then
+			warning_string="User/Hostname does not match: ${current_user_hostname}";
+		fi
+		# check for creation date diff
+		if [ "${rotate_date}" != "${current_date}" ]; then
+			if [ ! -z "${warning_string}" ]; then
+				warning_string="${warning_string}, ";
+			fi
+			warning_string="${warning_string}${rotate_date} != ${current_date}";
+		fi
+		# print out warning or ok
+		if [ ! -z "${warning_string}" ]; then
+			echo "[WARNING] ${warning_string}";
+			WARNING=1;
+		else
+			echo "[OK]";
+		fi
+	else
+		echo ""
+		echo "[!] FAILURE: ${response}";
+		ERROR=1;
+	fi
+
+done
+
+if [ ${WARNING} -eq 1 ]; then
+	echo "A warning has been found by either username/hostname string in the PEM key not matching to call username/hostname or the rotate date differs to the creation date from the PEM public key";
+fi
+if [ ${ERROR} -eq 1 ]; then
+	echo "An error has been found. Some connection did not success, investigate and fix before removing old ssh keys";
+fi
+
+# __END__