#!/usr/bin/bash

# Options must precede the positional arguments.
csd_extra_opts=()
while [ "$#" -gt 0 ]; do
	case "$1" in
	--no-reread)
		# Zero a failed replication fragment as a whole instead of
		# re-reading it page by page.
		csd_extra_opts+=(--manual-rec-no-reread)
		shift
		;;
	--)
		shift
		break
		;;
	-*)
		echo "Unknown option: $1" >&2
		exit 1
		;;
	*)
		break
		;;
	esac
done

CLUSTER=$1
CS_ID=$2

DAEMON_DIR=""

DIAG_DIR_BASE="/var/lib/vstorage/cs"
DIAG_DIR="$DIAG_DIR_BASE/$CLUSTER/$CS_ID"
UUID_FILE="$DIAG_DIR/uuid"
MOUNT_INFO_FILE="$DIAG_DIR/mount_info"
FSCK_OUT="$DIAG_DIR/fsck.out"
TRUNCATED_INODES="$DIAG_DIR/truncated_inodes"
CS_MODE_PATH="$DIAG_DIR/cs_mode"

RECOVERED_DIR=""
RECOVERED_LOGS=""
RECOVERED_TRASH=""
RECOVERED_JOURNAL_DIR=""
RECOVERED_JOURNAL=""

BASE_DEV=""
OVERLAY=""
NBD_DEV=""
mount_point=""
keep_overlay=0

CS_MODE_RW=0
CS_MODE_RO=1
CS_MODE_AUTO_REC=2
CS_MODE_MANUAL_REC=4
CS_MODE_FAIL_NO_REC=8
CS_MODE_SSD=16
CS_MODE_CSUM_DISABLED=32
CS_JOURNAL_SB_BAD=64
CS_RECOVERY_WITH_JOURNAL=128
CS_RECOVERY_POSSIBLE=256

cleanup()
{
	if [ "$keep_overlay" = 1 ]; then
		echo "Leaving overlay in place. To clean up after recovery:"
		echo "  pkill -f 'csd.*-r $RECOVERED_DIR'"
		echo "  umount $mount_point"
		echo "  qemu-nbd --disconnect $NBD_DEV"
		echo "  rm -f $OVERLAY"
		return
	fi

	if [ -n "$DAEMON_DIR" ] && csd_running_on "$DAEMON_DIR"; then
		echo "Live CSD still running; cleanup will not unmount" >&2
		return
	fi

	sync

	if csd_running_on "$RECOVERED_DIR"; then
		pkill -f "csd.*-r $RECOVERED_DIR" || true
	fi

	if [ -n "$mount_point" ] && [ -n "$NBD_DEV" ] &&
		[ "$(findmnt -rno SOURCE -M "$mount_point" 2>/dev/null)" = "$NBD_DEV" ]; then
		umount -- "$mount_point" >/dev/null 2>&1 || true
	fi

	if [ -n "$NBD_DEV" ] && [ -b "$NBD_DEV" ]; then
		qemu-nbd --disconnect "$NBD_DEV" >/dev/null 2>&1 || true
	fi

	if [ -n "$OVERLAY" ]; then
		rm -f -- "$OVERLAY"
	fi
}

load_mode_flags()
{
	local mode=0

	ro_state=0
	auto_rec_state=0
	manual_rec_state=0
	on_ssd=0
	csum_disabled=0
	journal_sb_bad=0
	recovery_with_journal=0
	recovery_possible=0

	if [ ! -f "$CS_MODE_PATH" ]; then
		return 0
	fi

	if ! read -r mode < "$CS_MODE_PATH"; then
		echo "Failed to read CS mode" >&2
		return 1
	fi

	if ! [[ "$mode" =~ ^[0-9]+$ ]]; then
		echo "Invalid mode: $mode" >&2
		return 1
	fi

	ro_state=$(( (mode & CS_MODE_RO) != 0 ))
	auto_rec_state=$(( (mode & CS_MODE_AUTO_REC) != 0 ))
	manual_rec_state=$(( (mode & CS_MODE_MANUAL_REC) != 0 ))
	on_ssd=$(( (mode & CS_MODE_SSD) != 0 ))
	csum_disabled=$(( (mode & CS_MODE_CSUM_DISABLED) != 0 ))
	journal_sb_bad=$(( (mode & CS_JOURNAL_SB_BAD) != 0 ))
	recovery_with_journal=$(( (mode & CS_RECOVERY_WITH_JOURNAL) != 0 ))
	recovery_possible=$(( (mode & CS_RECOVERY_POSSIBLE) != 0 ))

	return 0
}

clear_mode_flag_all()
{
	if ! rm -f -- "$CS_MODE_PATH"; then
		echo "failed to remove $CS_MODE_PATH" >&2
		return 1
	fi

	return 0
}

set_mode_flag()
{
	local flag="$1"
	local mode=0

	if [ -f "$CS_MODE_PATH" ]; then
		if ! read -r mode < "$CS_MODE_PATH"; then
			echo "Failed to read cs mode" >&2
			return 1
		fi
	fi

	if ! [[ "$mode" =~ ^[0-9]+$ ]]; then
		echo "Invalid mode in file: $mode" >&2

		if ! clear_mode_flag_all; then
			return 1
		fi
	fi

	mode=$(( mode | flag ))

	if ! echo "$mode" > "$CS_MODE_PATH"; then
		echo "Failed to write cs mode" >&2
		return 1
	fi

	return 0
}

clear_mode_flag()
{
	local flag="$1"
	local mode=0

	if [ -f "$CS_MODE_PATH" ]; then
		if ! read -r mode < "$CS_MODE_PATH"; then
			echo "Failed to read cs mode" >&2
			return 1
		fi
	fi

	if ! [[ "$mode" =~ ^[0-9]+$ ]]; then
		echo "Invalid mode in file: $mode" >&2
		return 1
	fi

	mode=$(( mode & ~flag ))

	if ! echo "$mode" > "$CS_MODE_PATH"; then
		echo "Failed to write cs mode" >&2
		return 1
	fi
}

check_prereq()
{
	for cmd in qemu-img qemu-nbd e2fsck findmnt blockdev modprobe mount; do
		if ! command -v "$cmd" >/dev/null 2>&1; then
			echo "Required command not found: $cmd" >&2
			return 1
		fi
	done
}

# PR_1_INODE_TOOBIG and PR_1_INLINE_DATA_NO_ATTR (e2fsck PROMPT_TRUNCATE).
extract_truncated_inodes()
{
	sed -nE \
		-e 's/.*Inode ([0-9]+) is too big\.[[:space:]]+(Truncate|TRUNCATED).*/\1/p' \
		-e 's/.*Inode ([0-9]+) has INLINE_DATA_FL flag but extended attribute not found\.[[:space:]]+(Truncate|TRUNCATED).*/\1/p' \
		"$FSCK_OUT" | sort -un >"$TRUNCATED_INODES"
}

read_uuid()
{
	if [ ! -r "$UUID_FILE" ]; then
		echo "UUID file is not readable" >&2
		return 1
	fi

	uuid=$(tr -d '\n' < "$UUID_FILE")
	if [ -z "$uuid" ]; then
		echo "UUID file is empty" >&2
		return 1
	fi
}

get_base_dev()
{
	read_uuid || return 1

	if ! BASE_DEV=$(blkid -U "$uuid"); then
		echo "Failed to resolve UUID $uuid" >&2
		return 1
	fi
}

csd_running_on()
{
	local want pid i
	local -a args
	want=$(realpath -m -- "$1") || return 1
	for pid in /proc/[0-9]*; do
		[ -r "$pid/cmdline" ] || continue
		mapfile -d '' -t args < "$pid/cmdline" || continue
		[ "${args[0]}" = "/usr/bin/csd" ] || continue
		for ((i = 1; i < ${#args[@]} - 1; i++)); do
			if [ "${args[i]}" = "-r" ]; then
				if [ "$(realpath -m -- "${args[i+1]}")" = "$want" ]; then
					return 0
				fi
			fi
		done
	done
	return 1
}

ensure_unmounted()
{
	local dev=$1

	if [ -n "$DAEMON_DIR" ] && csd_running_on "$DAEMON_DIR"; then
		echo "Live CSD is running (-r $DAEMON_DIR); leaving $dev mounted" >&2
		return 1
	fi

	if findmnt -rn -S "$dev" >/dev/null; then
		if ! umount -- "$dev"; then
			echo "Failed to unmount $dev" >&2
			return 1
		fi
	fi
}

connect_first_free_nbd()
{
	local dev

	for dev in /dev/nbd[0-9] /dev/nbd[0-9][0-9]; do
		[ -b "$dev" ] || continue
		[ -z "$(cat /sys/block/$(basename "$dev")/pid 2>/dev/null)" ] || continue
		qemu-nbd --connect="$dev" --format=qcow2 "$OVERLAY" || continue
		NBD_DEV=$dev
		return 0
	done

	echo "No free NBD device found" >&2
	return 1
}

read_mount_point()
{
	local dumped_opts=""

	if [ ! -r "$MOUNT_INFO_FILE" ]; then
		echo "mount_info is not readable: $MOUNT_INFO_FILE" >&2
		return 1
	fi

	if ! read -r mount_point dumped_opts < "$MOUNT_INFO_FILE"; then
		echo "Failed to read mount_info" >&2
		return 1
	fi

	if [ -z "$mount_point" ]; then
		echo "mount_info has empty mount point" >&2
		return 1
	fi
}

mount_nbd_rw()
{
	if [ ! -d "$mount_point" ]; then
		echo "Mount point does not exist: $mount_point" >&2
		return 1
	fi

	if findmnt -rno TARGET "$mount_point" >/dev/null; then
		echo "Mount point is already in use: $mount_point" >&2
		return 1
	fi

	if ! mount -t ext4 -o rw,relatime,data=ordered -- "$NBD_DEV" "$mount_point"; then
		echo "Failed to mount $NBD_DEV on $mount_point" >&2
		return 1
	fi

	echo "Mounted $NBD_DEV on $mount_point (rw,relatime,data=ordered)"
}

qemu_img_create()
{
	qemu-img create \
		-f qcow2 \
		-F raw \
		-b "$BASE_DEV" \
		"$OVERLAY"
}

fill_daemon_dir()
{
	local svc="/usr/lib/systemd/system/vstorage-csd.${CLUSTER}.${CS_ID}.service"

	if [[ -f "$svc" ]]; then
		dir=$(sed -n 's/.*DAEMON_DIR=\([^"[:space:]]*\).*/\1/p' "$svc" | head -1)
		if [[ -n "$dir" ]]; then
			printf '%s\n' "$dir"
			return 0
		fi
	fi

	return 1
}

fill_recovery_paths()
{
	RECOVERED_DIR="$DIAG_DIR_BASE/$CLUSTER/$CS_ID/recovered"
	RECOVERED_LOGS="$RECOVERED_DIR/logs"
	RECOVERED_TRASH="$RECOVERED_DIR/trash"
	RECOVERED_JOURNAL_DIR="$RECOVERED_DIR/journal"
	RECOVERED_JOURNAL="$RECOVERED_JOURNAL_DIR/journal"
}

journal_fiemap_rec()
{
	if /usr/bin/vstorage -c "$CLUSTER" recover-cs -e "$CS_ID"; then
		ln -s -- "$DIAG_DIR"/extracted.journal "$RECOVERED_JOURNAL" || return 1
	else
		return 1
	fi
}

create_chunk_links()
{
	local i hex target_path link_path

	for i in {0..255}; do
		printf -v hex '%02x' "$i"

		target_path="$DAEMON_DIR/$hex"
		link_path="$RECOVERED_DIR/$hex"

		if ! ln -s -- "$target_path" "$link_path"; then
			echo "failed to create symlink: $link_path -> $target_path" >&2
			return 1
		fi
	done

	return 0
}

prepare_cs_env()
{
	local rc=0
	local real=""

	real=$(realpath -m "$RECOVERED_DIR") || return 1

	case "$real" in
		"$DIAG_DIR_BASE"/*) ;;
		*)
			echo "Path not under $DIAG_DIR_BASE: $real" >&2
			return 1
			;;
	esac

	rm -rf -- "$real" || return 1

	install -d -m 0755 "$real" || return 1

	install -d -m 0755 "$RECOVERED_LOGS" || return 1
	install -d -m 0755 "$RECOVERED_TRASH" || return 1

	rsync -a --delete -- "$DAEMON_DIR"/control "$RECOVERED_DIR/" || return 1
	rm -rf -- "$RECOVERED_DIR"/control/journal || return 1

	create_chunk_links || return 1

	return 0
}

prepare_cs_journal()
{
	local journal_link="$RECOVERED_DIR/control/journal"

	if ! install -d -m 0755 "$RECOVERED_JOURNAL_DIR"; then
		return 1
	fi

	if ! ln -s -- "$RECOVERED_JOURNAL_DIR" "$journal_link"; then
		rm -rf -- "$RECOVERED_JOURNAL_DIR"
		return 1
	fi

	if ! ln -s -- "$DAEMON_DIR"/control/journal/checksum "$RECOVERED_JOURNAL_DIR"/checksum; then
		rm -rf -- "$RECOVERED_JOURNAL_DIR"
		rm -f -- "$journal_link"
		return 1
	fi

	if ! journal_fiemap_rec; then
		rm -rf -- "$RECOVERED_JOURNAL_DIR"
		rm -f -- "$journal_link"
		return 1
	fi
}

# The MDS logs "CS#<id> is active" only on an inactive->active transition.
# A recovery csd started seconds after the live one stopped goes
# unstable->active instead and never produces that event, so ask the master
# MDS for the activity status directly.
cs_is_active_on_mds()
{
	/usr/bin/vstorage -c "$CLUSTER" raw-stat -M -C "$CS_ID" 2>/dev/null |
		awk '$1 == "MDS.cs.act_status" { found = 1; exit ($2 != "active") }
		     END { if (!found) exit 1 }'
}

start_cs()
{
	local waited=0
	local timeout="${CSD_START_TIMEOUT:-300}"

	(
		export CLUSTER CS_ID
		cd "$RECOVERED_DIR" || exit 1
		exec nohup /usr/bin/csd \
			"${csd_extra_opts[@]}" \
			-r "$RECOVERED_DIR" \
			--log-dir "$RECOVERED_LOGS" \
			--log-prefix "cs" \
			--log-compression zst \
			-L 99x100M
	) &

	CSD_PID=$!
	echo "CSD PID: $CSD_PID; waiting for CS#$CS_ID to become active on MDS"

	while true; do
		if ! kill -0 "$CSD_PID" 2>/dev/null; then
			echo "CSD exited before startup completed" >&2
			return 1
		fi

		if cs_is_active_on_mds; then
			echo "CS#$CS_ID is active on MDS"
			return 0
		fi

		if [ "$waited" -ge "$timeout" ]; then
			echo "CS#$CS_ID did not become active on MDS within ${timeout}s" >&2
			return 1
		fi

		echo "CSD startup is in process.."
		sleep 5
		waited=$((waited + 5))
	done
}

main()
{
	local uuid=""
	local ro_state=0
	local auto_rec_state=0
	local manual_rec_state=0
	local on_ssd=0
	local csum_disabled=0
	local journal_sb_bad=0
	local recovery_with_journal=0
	local recovery_possible=0

	if [ "$#" -ne 2 ]; then
		echo "Usage: $0 [--no-reread] <cluster> <cs_id>" >&2
		exit 1
	fi

	if [ "$(id -u)" -ne 0 ]; then
		echo "Should be run as root" >&2
		exit 1
	fi

	check_prereq || exit 1
	DAEMON_DIR=$(fill_daemon_dir) || exit 1

	if [ ! -d "$DIAG_DIR" ]; then
		echo "CS recovery is not possible: DIAG_DIR missing. Most likely unsupported CS version."
		exit 1
	fi

	fill_recovery_paths
	if csd_running_on "$RECOVERED_DIR"; then
		echo "Another manual recovery instance for this CS is running"
		exit 1
	fi

	if ! load_mode_flags; then
		echo "Failed to load recovery flags"
		exit 1
	fi

	if [ "$recovery_possible" -eq 0 ]; then
		echo "CS manual recovery is not possible. Conditions are that prevent it:"

		if [[ ! -f "$UUID_FILE" || ! -s "$UUID_FILE" ]]; then
			echo "UUID_FILE $UUID_FILE doesn't exist or empty"
		fi

		if [[ ! -f "$MOUNT_INFO_FILE" || ! -s "$MOUNT_INFO_FILE" ]]; then
			echo "MOUNT_INFO_FILE $MOUNT_INFO_FILE doesn't exist or empty"
		fi

		exit 1
	fi

	get_base_dev || exit 1

	OVERLAY=$(mktemp --tmpdir=/var/tmp \
		"$(basename "$BASE_DEV").repair.XXXXXX.qcow2") || exit 1

	trap cleanup EXIT
	trap 'exit 1' INT TERM

	rm -f "$OVERLAY"

	ensure_unmounted "$BASE_DEV" || exit 1

	qemu_img_create || exit 1

	modprobe nbd max_part=16
	connect_first_free_nbd || exit 1

	LC_ALL=C e2fsck -f -y "$NBD_DEV" 2>&1 | tee "$FSCK_OUT"
	fsck_rc=${PIPESTATUS[0]}

	printf '\nE2fsck exit code: %d\n' "$fsck_rc" | tee -a "$FSCK_OUT"

	if [ "$fsck_rc" -gt 2 ]; then
		exit 1
	fi

	extract_truncated_inodes

	read_mount_point || exit 1
	mount_nbd_rw || exit 1

	prepare_cs_env || exit 1
	if [ "$recovery_with_journal" -eq 1 ] && prepare_cs_journal; then
		echo "CS manual recovery with journal. Some chunks can still be healthy."
	else
		echo "CS manual recovery without journal. All recovered chunks will be marked corrupted."
	fi

	chown -R root:root "$RECOVERED_DIR"
	if ! set_mode_flag "$(( CS_MODE_MANUAL_REC | CS_MODE_RO ))"; then
		echo "Failed to set manual recovery flag for CS"
		exit 1
	fi

	start_cs || exit 1

	keep_overlay=1
	return 0
}

main "$@"
