#!/bin/bash
# Problem report helper script
PROG="$0"
LOCAL=no
FULL_REPORT=yes
FNAME=""
CLUSTER_NAME=""
DIR=""
CFG_DIR="/etc/vstorage"
CTL_TOOL="/usr/bin/vstorage"
CLUSTERS_DIR="/etc/vstorage/clusters"
MAKE_REPORT="/usr/bin/vstorage-make-report"
BLOG_READER="/usr/bin/blogcat"
ZSTDCAT="/usr/bin/zstdcat"
SSH_OPTS="-o PasswordAuthentication=no -o StrictHostKeyChecking=no"
LOG_LINES=100000

cleanup() {
	rm -rf "$DIR"
}

prepare_logs() {
	local LOGDIR="$1"
	local SERVICE="$2"
	local LOGTAIL="$LOGDIR/$SERVICE.log.tail"
	local LINES_LEFT="$LOG_LINES"
	rm -f "$LOGTAIL"
	logs=$(find "$LOGDIR" \( -type f -iname "$SERVICE"\*.blog -o -iname "$SERVICE"\*.zst \) -exec ls -t {} \; | sort -n)
	for l in $logs; do
		if [[ -s "$l" ]]; then
			if [[ "$l" == *"blog"* ]]; then
				$BLOG_READER "$l" 2>&1 | tail -n "$LINES_LEFT" > "$LOGTAIL.1"
			else
				$ZSTDCAT "$l" 2>&1 | tail -n "$LINES_LEFT" > "$LOGTAIL.1"
			fi
		fi
		GOT_LINES=$(wc -l "$LOGTAIL.1" | awk '{print $1}')
		if [ "$GOT_LINES" -gt 0 ]; then
			LINES_LEFT=$((LINES_LEFT - GOT_LINES))
			mv "$LOGTAIL" "$LOGTAIL.2" 2>/dev/null
			cat "$LOGTAIL.1" "$LOGTAIL.2" > "$LOGTAIL" 2>/dev/null
			rm -f "$LOGTAIL.2" 2>/dev/null
		fi
		rm -f "$LOGTAIL.1"
		[ "$LINES_LEFT" -eq 0 ] && break
	done
	[ -s "$LOGTAIL" ] && echo "$LOGTAIL"
}

for_cluster() {
	local OP="$1"
	local SERVICE="$2"
	local CLUSTER_DIR="${CLUSTERS_DIR}/$CLUSTER_NAME"
	cat "${CLUSTER_DIR}/$SERVICE.list" 2>/dev/null | grep -v '^#' | grep -v '^\s*$' | while read -r dir; do
		log_dir=$(readlink "$dir/logs")
		("$OP" "${log_dir}" "$SERVICE")
	done
}

get_mounts_info()
{
        local tmp_dir=$1
        mounts=`cat /proc/self/mounts | awk '{if ($1 == "vstorage://'"$CLUSTER_NAME"'" && $3 == "fuse.vstorage") printf("%s/.pstorage.info\n", $2)}'`
        [ "x$mounts" = "x" ] && return 0
        for m in $mounts; do
                mkdir -p "$tmp_dir/$m"
                for f in `ls $m`; do
                        cat "$m/$f" > "$tmp_dir/$m/$f"
                done
        done
}

prepare_file_list() {
CUSTOM_WATCHDOG_LOG=`cat /sys/kernel/watchdog_log_path 2>/dev/null`
	cat <<EOF
$CFG_DIR/auth.modules
$CFG_DIR/mds.config
$CFG_DIR/cs.config
$CFG_DIR/host_id
$CLUSTERS_DIR/$CLUSTER_NAME/mds.list
$CLUSTERS_DIR/$CLUSTER_NAME/cs.list
`[ -f "/fence_wdog.log" ] && echo /fence_wdog.log`
`[ -f "/var/log/vstorage/vstorage-target.log" ] && echo /var/log/vstorage/vstorage-target.log`
`[ -f "/var/log/vstorage/vstorage-target-manager.log" ] && echo /var/log/vstorage/vstorage-target-manager.log`
`[ -f "/var/log/vstorage/vstorage-target-monitor.log" ] && echo /var/log/vstorage/vstorage-target-monitor.log`
`[ -f "$CUSTOM_WATCHDOG_LOG" ] && [ "x$CUSTOM_WATCHDOG_LOG" != "x/fence_wdog.log" ] && echo $CUSTOM_WATCHDOG_LOG`
`[ -f "$CLUSTERS_DIR/$CLUSTER_NAME/mds.config" ] && echo $CLUSTERS_DIR/$CLUSTER_NAME/mds.config`
`[ -f "$CLUSTERS_DIR/$CLUSTER_NAME/bs.list" ] && echo $CLUSTERS_DIR/$CLUSTER_NAME/bs.list`
`[ -f "$CLUSTERS_DIR/$CLUSTER_NAME/cs.config" ] && echo $CLUSTERS_DIR/$CLUSTER_NAME/cs.config`
/root/.vstorage/$CLUSTER_NAME
EOF
	if [ "x$FULL_REPORT" = "xyes" ] ; then
		for_cluster prepare_logs mds
		for_cluster prepare_logs cs
		prepare_logs "/var/log/vstorage/$CLUSTER_NAME" vstorage-mount
	fi
}

get_history() {

	mkdir -p "$DIR/history"
	$CTL_TOOL -c "$CLUSTER_NAME" stat -X -F list 2>/dev/null | tr '\n' ' ' | sed "s#\s##g" | sed "s#<\/entry>#<\/entry>\n#g" | sed "s#.*<entry>#\n<entry>#" | grep "<entry>" | sed "s#.*name>\(.*\)<\/name.*nr_levels>\(.*\)<\/nr_levels.*#\1 \2#g" | while read VALUE NUM_INTERVALS
	do
		for num in `seq 0 $[NUM_INTERVALS-1]` ; do
			$CTL_TOOL -c "$CLUSTER_NAME" stat -X -F "$VALUE:$num" > "$DIR/history/$VALUE:$num" 2>/dev/null
	        done
	done
}

local_work() {
	get_mounts_info "$DIR/"
	[ -e /var/log/vstorage/abgw.log.zst ] && cp /var/log/vstorage/abgw.log.zst "$DIR/"
	ostor_logs=$(ls /var/log/ostor/*.gz 2> /dev/null | wc -l)
	[ "$ostor_logs" -gt 0 ] && for i in /var/log/ostor/*.gz; do zcat $i | tail -n "$LOG_LINES" > "$DIR/"$(basename $i).tail ; done
	prepare_file_list | tar --ignore-failed-read -cz -C "$DIR" . -C / -T -
	# check tar exit code: 2 = failure
	[ $? -ne 2 ]
}

remote_work() {
	if ! $CTL_TOOL -c "$CLUSTER_NAME" stat --xml >"$DIR/stat.xml" 2>/dev/null; then
		echo "Failed to get information about the cluster"
		return 1
	fi
	$CTL_TOOL -c "$CLUSTER_NAME" get-event >"$DIR/event.log" 2>/dev/null
	$CTL_TOOL -c "$CLUSTER_NAME" get-config > "$DIR/cluster.cfg" 2>/dev/null
	$CTL_TOOL -c "$CLUSTER_NAME" view-license | grep -v owner_name > "$DIR/license" 2>/dev/null
	$CTL_TOOL -c "$CLUSTER_NAME" view-license -a -X | grep -v owner_name > "$DIR/licenses.xml" 2>/dev/null
	[ $? -ne 0 ] && rm -f "$DIR/licenses.xml"

	get_history

	if [ "x$FULL_REPORT" = "xyes" ] ; then

	for host in $(cat "$DIR/stat.xml" | grep -o '<host>.*</host>' | sed -e 's/<host>\(.*\)<\/host>/\1/' -e 's/:[0-9]*$//' | sort -u); do
		/sbin/ip a 2>/dev/null | grep $host 2>&1 > /dev/null
		if [ $? -ne 0 ] ; then
			ssh $SSH_OPTS root@"$host" "$MAKE_REPORT" --local "$CLUSTER_NAME" >"$DIR/$host.tgz" 2>"$DIR/$host.log" &&
			echo "Collected report from host $host." ||
			echo "Failed to collect report from host $host."
		else
			"$MAKE_REPORT" --local "$CLUSTER_NAME" >"$DIR/$host.tgz" 2>"$DIR/$host.log" &&
                        echo "Collected report from host $host." ||
                        echo "Failed to collect report from host $host."
		fi
	done

	fi # ["x$FULL_REPORT" = "xyes"]


	if [ ! "$FNAME" ]; then
		DATE=$(date +"%Y%m%d-%H%M%S")
		FNAME=vstorage-report-"$DATE".tgz
	fi
	tar -czf "$FNAME" -C "$DIR" .
	echo
	echo "The report is created and saved to $FNAME."
	echo "Please send the report to your product support team."
}

usage() {
	echo "Usage: $PROG [-f FILE] CLUSTER_NAME" 1>&2
	exit 1
}

if ! options=$(getopt -o lSf:h -l local,file:,help -- "$@")
then
	# something went wrong, getopt will put out an error message for us
	exit 1
fi

eval set -- "$options"

while [ $# -gt 0 ]; do
	case $1 in
		-S|--short) FULL_REPORT="no" ;;
		-l|--local) LOCAL="yes" ;;
		-f|--file) FNAME="$2" ; shift;;
		-h|--help) usage ;;
		(--) shift; break;;
		(-*) echo "$PROG: error - unrecognized option $1" 1>&2; exit 1;;
		(*) break;;
	esac
	shift
done

[ $# -ne 1 ] && usage
CLUSTER_NAME="$1"

trap cleanup EXIT
DIR=$(mktemp -d)
if ! [ "$DIR" ]; then
	echo "Failed to create temporary directory" 1>&2
	exit 1
fi

if [ "$LOCAL" = "yes" ]; then
	local_work
else
	remote_work
fi
