#!/usr/bin/bash
#
# Drops every S3 endpoint VIP of this node on FENCE, and puts back what shaman
# assigns this node on UNFENCE, retrying in the background when that fails.
# Run by shaman-monitor with the other fence hooks.
#
# Provided environment variables:
# - EVENT - FENCE or UNFENCE
#

. /usr/share/shaman/functions

verify_caller

S3EP=/usr/bin/ostor-s3ep
SYSTEMCTL=/usr/bin/systemctl
RETRY_UNIT=ostor-s3ep-unfence-retry.service

# A unit of that name is loaded while it runs, waits to restart or stops.
unit_loaded() {
	[ "$($SYSTEMCTL show -p LoadState --value "$1" 2>/dev/null)" = loaded ]
}

# Retry restore 30 s after each failed run, for up to 10 minutes after the retry
# starts, timed on /proc/uptime.
# systemd-run expands ${...} and $$ in its command line: the loop uses neither.
# shellcheck disable=SC2016 # the unit's bash expands the loop, not this one
RETRY_LOOP='
read -r -d . start < /proc/uptime
while :; do
	/usr/bin/sleep 30
	/usr/bin/ostor-s3ep endpoint restore
	rc=$?
	[ $rc -eq 0 ] && exit 0
	if [ $rc -ne 2 ] && [ $rc -lt 128 ]; then
		echo "The retry stops: ostor-s3ep endpoint restore exited $rc" 1>&2
		exit $rc
	fi
	read -r -d . now < /proc/uptime
	if [ $((now - start + 30)) -gt 600 ]; then
		echo "The retry gives up after 10 minutes: run ostor-s3ep endpoint restore by hand" 1>&2
		exit 1
	fi
done'

# The bound leaves the last run, which starts within 10 minutes, five more.
start_retry() {
	unit_loaded $RETRY_UNIT && return 0
	/usr/bin/systemd-run --no-block --collect --quiet --unit=$RETRY_UNIT \
		--description="Retry of the S3 endpoint restore after UNFENCE" \
		-p RuntimeMaxSec=15min /bin/bash -c "$RETRY_LOOP" ||
		echo "Cannot start $RETRY_UNIT: the S3 endpoints stay down until the next UNFENCE" 1>&2
}

case "$EVENT" in
FENCE)
	# A retry decides each AddVIP on a fresh read, but an AddVIP between that
	# read and the fence would re-create the bridge: stop it first.
	unit_loaded $RETRY_UNIT && $SYSTEMCTL stop $RETRY_UNIT
	$S3EP fence
	exit $?
	;;
UNFENCE)
	$S3EP is-cluster-member --quiet
	case $? in
	0)	;;
	3)	exit 0 ;;	# a node that lost its name file must not plumb
	*)	echo "Cannot check ostor cluster membership: restoring anyway" 1>&2 ;;
	esac
	$S3EP endpoint restore
	rc=$?
	if [ $rc -eq 2 ] || [ $rc -ge 128 ]; then
		start_retry
	fi
	exit 0
	;;
"")
	echo "EVENT environment variable undefined" 1>&2
	exit 1
	;;
*)
	echo "Unknown event $EVENT" 1>&2
	exit 1
	;;
esac
