#!/bin/bash

CTL_TOOL="vstorage-iscsi"
ISCSI_ETC="/etc/vstorage/iscsi"
ISCSI_SHM="/dev/shm/vstorage-iscsi"
ISCSI_TMP="/dev/shm/vstorage-iscsi"
PSTORAGE_ETC="/etc/vstorage"
PLOOP="/usr/sbin/ploop"
SHAMAN="/usr/sbin/shaman -q"
VSTORAGE_BIN="/usr/bin/vstorage"
SYSTEMD_TOOL="/usr/bin/systemctl"
OVSCTL_CMD="/usr/bin/ovs-vsctl"
VSTORAGE_TARGET_BIN="/usr/bin/vstorage-target"

TARGET_NO_OWNER="unregistered"

source /usr/libexec/vstorage-iscsi/vstorage-scsi-target-tgtd
source /usr/libexec/vstorage-iscsi/vstorage-scsi-target-tcm
source /usr/libexec/vstorage-iscsi/iscsi_functions
source /usr/libexec/vstorage-iscsi/fcoe_functions

function echo_fsync {
    dd of=$1 conv=fsync >/dev/null 2>&1
}

# `skip_target_checking` param prevents full target checking
function pcs_iscsi_set_env() {
	local target="$1"
	local skip_target_checking="$2"
	local tgt=""
	local transport=""

	if [ ! -f "$ISCSI_ROOT/$target/control/metadata" ]; then
			tgt="tgtd"
			transport="iscsi"
	else
		read 2>/dev/null tgt transport <"$ISCSI_ROOT/$target/control/metadata"
		if [ $? -ne 0 ] ; then
			echo "Can't get \"$target\" metadata from the file \"$ISCSI_ROOT/$target/control/metadata\"" 1>&2
			return 1
		fi
	fi

	if [ "$tgt" != "tcm" -a "$tgt" != "tgtd" ] ; then
		echo "Unsupported scsi engine \"$tgt\"" 1>&2
		return 2
	fi

	SCSI_TGT=$tgt
	if [ ! -z "$skip_target_checking" ] ; then
		SCSI_TRANSPORT=$transport
		${SCSI_TGT}_check_scsi_transport
		[ $? -ne 0 ] && return 3
		return 0
	fi

	${SCSI_TGT}_set_transport transport=$transport target=$target
	return $?
}

function pcs_icsi_get_meta {
	local target="$1"
	local meta=""

	if [ ! -f "$ISCSI_ROOT/$target/control/metadata" ]; then
		meta="tgtd iscsi"
	else
		meta=`cat $ISCSI_ROOT/$target/control/metadata`
	fi

	echo $meta
}

function is_root_on_vstorage {
	for d in $(cat /proc/mounts | awk '/^[pv]storage:\/\//{ print $2}'); do
		[[ "$ISCSI_ROOT" == $d* ]] && return 1
	done
	return 0
}

function pcs_iscsi_set_storage_attr {
	local dir="$1"; shift
	$VSTORAGE_BIN "set-attr" "$dir" ${@} >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Can't set storage attributes for $dir" 1>&2
		return 1
	fi
	return 0
}

function get_host_id {
	cat $PSTORAGE_ETC/host_id
	rc=$?
	if [ $? -ne 0 ] ; then
		echo "Can't read host id" 1>&2
		return $rc
	fi
	return 0
}

function pcs_iscsi_is_registered_local {
	local target="$1"
	local host_id="$2"
	local owner=""
	if [ -z "$host_id" ] ; then
		host_id=`get_host_id`
		[ $? -ne 0 ] && return 2
	fi

	owner=`cat $ISCSI_ROOT/$target/control/host`
	if [ $? -ne 0 ] ; then
		echo "Unable read file $target/control/host" 1>&2
		return 3
	fi

	[ -z "$owner" -o "$owner" != "$host_id" ] && return 1

	return 0
}

function get_id {
	[ ! -d $ISCSI_SHM ] && mkdir -p $ISCSI_SHM

	(
	flock -x 200
	last_tid=`ls $ISCSI_SHM 2>/dev/null | sort -g -r | head -n 1`
	if [ -z "$last_tid" ] ; then
		last_tid="10"
	else
		last_tid=$[ $last_tid + 1]
	fi
	touch $ISCSI_SHM/$last_tid /dev/null 2>&1
	echo $last_tid
	) 200>$ISCSI_SHM/.lock
}

function pcs_iscsi_init_host {
	[ ! -d $ISCSI_SHM ] && mkdir -p $ISCSI_SHM/.running >/dev/null 2>&1

	iscsi_init_host && fc_init_host && fcoe_init_host
	return $?
}

function put_id {
	local tid="$1"
	local target="$2"
	(
		flock -x 201
		rm -f $ISCSI_SHM/$tid
	) 201>$ISCSI_SHM/.lock
	[ -n "$target" ] && rm -f "$ISCSI_SHM/.running/$target"
}

function pcs_iscsi_list_registered {
	host_id=`get_host_id`
	[ $? -ne 0 ] && return 1

	ls $ISCSI_ROOT 2>/dev/null | grep "^iqn*" 2>/dev/null | while read target
	do
		[ ! -d  "$ISCSI_ROOT/$target/control" ] && continue

		pcs_iscsi_is_registered_local $target $host_id >/dev/null 2>&1
		[ $? -ne 0 ] && continue
		echo ""
		echo "iscsi-$target"
		echo "$ISCSI_ROOT/$target"
	done
}

function pcs_iscsi_target_has_owner {
	local target="$1"
	local owner=""
	if [ -f "$ISCSI_ROOT/$target/control/host" ]; then
		owner=`cat $ISCSI_ROOT/$target/control/host`
		if [ $? -ne 0 ] ; then
			echo "Unable read file $ISCSI_ROOT/$target/control/host" 1>&2
			return 3
		fi
	fi

	[ -n "$owner" -a "$owner" != "$TARGET_NO_OWNER" ] && return 1
	return 0
}

# TODO: this function isn't used by external caller,
# so it makes sense to remove this call from public API
# otherwise add call pcs_iscsi_set_env inside
function pcs_iscsi_bind_account {
	local target="$1"
	local ctl_port="$2"
	local account=""

	account_file="$ISCSI_ROOT/$target/control/account"
	[ ! -f $account_file ] && return
	user=$(cat $account_file)
	if [ -n "$user" ] ; then
		local passw=$($VSTORAGE_TARGET_BIN account-getpass -user $user)
		if [ $? -ne 0 ] ; then
			echo "Failed to get $user password" 1>&2
			return 1
		fi
		${SCSI_TGT}_add_account target=$target user=$user passw=$passw ctl_port=$ctl_port
		return $?
	fi
	return 0
}

function pcs_iscsi_unregister_target_ha {
	local target="$1"
	local revoke="$2"

	if [ -n "$revoke" ] ; then
		# cleanup old leases after MDS master changed
		$VSTORAGE_BIN revoke "$ISCSI_ROOT/$target/control/host" 1>&2
	fi

	msg=`echo "$TARGET_NO_OWNER" | echo_fsync "$ISCSI_ROOT/$target/control/host"`
	if [ $? -ne 0 ]; then
		echo "$msg" 1>&2
		echo "Can't write in $ISCSI_ROOT/$target/control/host" 1>&2
		return 2
	fi
	return 0
}

function pcs_iscsi_unregister_target {
	local target="$1"

	host_id=`get_host_id`
	[ $? -ne 0 ] && return 1

	prev_owner=`cat "$ISCSI_ROOT/$target/control/host" 2>/dev/null`

	pcs_iscsi_unregister_target_ha "$target"
	[ $? -ne 0 ] && return 3

	[ "$prev_owner" = "$host_id" ] && $SHAMAN del "iscsi-$target" >/dev/null 2>&1
	return 0
}

function pcs_iscsi_register_target_ha {
	local target="$1"
	local host_id
	host_id=`get_host_id`
	[ $? -ne 0 ] && return 1

	# save host id in $ISCSI_ROOT/$target/control/host
	msg=`echo $host_id | echo_fsync "$ISCSI_ROOT/$target/control/host"`
	if [ $? -ne 0 ]; then
		echo "$msg" 1>&2
		echo "Unable save host_id in $ISCSI_ROOT/$target/control/host" 1>&2
		return 2
	fi

	# reset running state after registration
	rm -f "$ISCSI_ROOT/$target/control/.running"
	return 0
}

function pcs_iscsi_filter_shaman_exit_code {
	# exit code 5 means "node is not registered in the HA cluster"
	[ $1 -ne 0 -a $1 -ne 5 ] && return $1
	return 0
}

function pcs_iscsi_register_target {
	local target="$1"
	local prev_owner=""

	if [ -f "$ISCSI_ROOT/$target/control/host" ]; then
		prev_owner=`cat "$ISCSI_ROOT/$target/control/host"`
		if [ $? -ne 0 ] ; then
			echo "Unable read $ISCSI_ROOT/$target/control/host" 1>&2
			return 1
		fi
	fi

	pcs_iscsi_register_target_ha "$target"
	[ $? -ne 0 ] && return 1

	if [ -z "$prev_owner" -o "$prev_owner" = "$TARGET_NO_OWNER" ]; then
		$SHAMAN add "iscsi-$target" -P $ISCSI_ROOT/$target >/dev/null
		pcs_iscsi_filter_shaman_exit_code $?
		if [ $? -ne 0 ] ; then
			echo "Can't register target '$target' as shaman's resource" 1>&2
			echo "Leave target in unregistered state." 1>&2
			pcs_iscsi_unregister_target "$target" >/dev/null 2>&1
			return 2
		fi
	else
		$SHAMAN move-from "iscsi-$target" "$prev_owner" >/dev/null
		pcs_iscsi_filter_shaman_exit_code $?
		if [ $? -ne 0 ] ; then
			echo "Can't register target '$target' as shaman's resource" 1>&2
			echo "Leave target in unregistered state." 1>&2
			pcs_iscsi_unregister_target "$target" >/dev/null 2>&1
			return 3
		fi
	fi

	return 0
}

function pcs_iscsi_attach_lun {
	local ctl_port="$1"
	local lun_id="$2"
	local ploop_dev="$3"
	local serial_sn="$4"
	local target="$5"

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 1

	${SCSI_TGT}_attach_lun target=$target dev=$ploop_dev lun_id=$lun_id serial_sn=$serial_sn ctl_port=$ctl_port
	rc=$?

	return $rc
}

function pcs_iscsi_get_ploop_dev {
	local target="$1"
	local lun="$2"
	local lun_id=`expr "$lun" : "^lun\([0-9]*\)" `
	local canon_path=$(readlink -f "$ISCSI_ROOT/$target/$lun/ploop")
	local ploop_dev=$( $PLOOP list | grep "$canon_path" | cut -d ' ' -f 1)
	if [ -z "$ploop_dev" ] ; then
		echo "Unable find device for $ISCSI_ROOT/$target/$lun/ploop" 1>&2
		return 1
	fi

	if [ ! -b "/dev/$ploop_dev" ] ; then
		echo "Unable find block device /dev/$ploop_dev" 1>&2
		return 2
	fi
	echo "/dev/$ploop_dev"
	return 0
}

function ploop_grow_lun {
	local target="$1"
	local lun="$2"
	local new_size="$3"
	local ctl_port="$4"
	local serial_sn=`pcs_iscsi_get_serial $target $lun`

	if [ -n "$ctl_port" ] ; then
		ploop_dev=`pcs_iscsi_get_ploop_dev $target $lun`
		[ $? -ne 0 ] && return 2
	fi

	# grow ploop
	msg=`$PLOOP grow -s $new_size "$ISCSI_ROOT/$target/$lun/DiskDescriptor.xml" 2>&1`
	if [ $? -ne 0 ] ; then
		echo "$msg" 1>&2
		echo "Can't grow ploop $ISCSI_ROOT/$target/$lun/ploop" 1>&2
		return 3
	fi

	if [ -n "$ploop_dev" ] ; then
		pcs_iscsi_set_env $target
		if [ $? -ne 0 ] ; then
			echo "Can't rescan lun after growing" 1>&2
			return 4
		fi

		${SCSI_TGT}_rescan_lun target=$target lun_id=$lun_id serial_sn=$serial_sn dev=$ploop_dev ctl_port=$ctl_port
		return $?
	fi

	return 0
}

function file_grow_lun {
	local target="$1"
	local lun="$2"
	local new_size="$3"
	local fpath=$ISCSI_ROOT/$target/$lun/disk.img

	msg=`truncate -s $new_size "$fpath" 1>&2`
	if [ $? -ne 0 ] ; then
		echo "$msg" 1>&2
		echo "Can't grow image $fpath" 1>&2
		return 1
	fi
	return 0
}

function pcs_iscsi_grow_lun {
	local target="$1"
	local lun="$2"
	local new_size="$3"
	local lun_id=`expr "$lun" : "^lun\([0-9]*\)" `
	local serial_sn=`pcs_iscsi_get_serial $target $lun`
	local ploop_dev=""
	local ctl_port=""

	if [ ! -d "$ISCSI_ROOT/$target/$lun" ] ; then
		echo "LUN $lun_id for target $target doesn't exist" 1>&2
		return 1
	fi

	ctl_port=`pcs_iscsi_get_ctl_port $target`

	local image_type=`pcs_iscsi_image_type "$ISCSI_ROOT/$target/$lun"`
	${image_type}_grow_lun "$target" "$lun" "$new_size" "$ctl_port" || return $?

	if [ -n "$ctl_port" ] ; then
		${SCSI_TGT}_rescan_lun target=$target lun_id=$lun_id serial_sn=$serial_sn dev=$ploop_dev ctl_port=$ctl_port
		return $?
	fi

	return 0
}

function ploop_image_mount {
	local target="$1"
	local lun="$2"
	local lun_dir=$3
	msg=`$PLOOP mount "$lun_dir/DiskDescriptor.xml" 2>&1`
	if [ $? -ne 0 ] ; then
		echo "$msg" 1>&2
		echo "Can't mount ploop $lun_dir/ploop" 1>&2
		return 2
	fi

	dev=`pcs_iscsi_get_ploop_dev $target $lun`
	[ $? -ne 0 ] && return 3
	echo $dev

	if [ -f "/sys/block/${dev:5}/ptune/discard_alignment" ]; then
		echo 4096 > "/sys/block/${dev:5}/ptune/discard_alignment"
		echo 4096 > "/sys/block/${dev:5}/ptune/discard_granularity"
	fi

	return 0
}

function file_image_mount {
	echo "$3/disk.img"
}

function file_image_umount {
	return
}

function ploop_image_umount {
	local lun_dir=$1
	$PLOOP umount "$lun_dir/ploop" >/dev/null 2>&1
}

function ploop_image_get_dev {
	local lun_dir="$1"

	ploop_full_path="$lun_dir/ploop"
	ploop_dev_name=`ploop list | grep "${ploop_full_path}" | cut -d ' ' -f1`

	echo $ploop_dev_name
}

function file_image_get_dev {
	echo "$1/disk.img"
}

function pcs_iscsi_up_lun {
	local target="$1"
	local ctl_port="$2"
	local lun="$3"
	local dev=""
	local msg=""
	local serial_sn=`pcs_iscsi_get_serial $target $lun`
	local lun_id=`expr "$lun" : "^lun\([0-9]*\)" `
	local lun_dir="$ISCSI_ROOT/$target/$lun"
	local image_type=`pcs_iscsi_image_type "$lun_dir"`

	if [ ! -d "$ISCSI_ROOT/$target/$lun" ] ; then
		echo "LUN $lun_id for target $target doesn't exist" 1>&2
		return 1
	fi

	dev=`${image_type}_image_mount $target $lun $lun_dir`
	ret=$?
	[ $ret -ne 0 ] && return $ret

	pcs_iscsi_attach_lun "$ctl_port" "$lun_id" "$dev" "$serial_sn" "$target"
	if [ $? -ne 0 ] ; then
		ploop_image_umount $lun_dir
		return 4
	fi

	return 0
}

# stop running target
function pcs_iscsi_down_target {
	local target="$1"
	local ctl_port="$2"
	local force="$3"
	local luns=""

	pcs_iscsi_is_registered_local "$target"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "Target $target not registered on current host" 1>&2
		return $rc
	fi

	if [ -z "$ctl_port" ]; then
		if [ ! -f "$ISCSI_SHM/.running/$target" ] ; then
			echo "Target $target not running" 1>&2
			return 0
		fi
		ctl_port=`cat $ISCSI_SHM/.running/$target`
	fi

	if [ -z "$ctl_port" ]; then
		echo "Control port is unknown" 1>&2
		return 1
	fi

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 2

	if [ -z "$force" ]; then
		# try failing early with a diagnostics on non-forced
		# stop, leaving the target online if there are active
		# connections.
		pcs_iscsi_target_can_stop_nonforcibly "$ctl_port" "$target"
		rc=$?
		if [ $rc -ne 0 ] ; then
			return $rc
		fi
	fi

	${SCSI_TGT}_delete_target target=$target force=$force ctl_port=$ctl_port
	rc=$?
	[ $rc -ne 0 ] && return $rc

	if [ -f "$ISCSI_ROOT/$target/control/address" ] ; then
		addresses=`cat "$ISCSI_ROOT/$target/control/address"`
		${SCSI_TRANSPORT}_remove_addr $addresses
	fi

	luns=`ls -d "$ISCSI_ROOT/$target"/lun[0-9]* 2>/dev/null`
	if [ -n "$luns" ] ; then
		for l in $luns; do
			local lun=$(basename $l)
			local lun_dir="$ISCSI_ROOT/$target/$(basename $l)"
			local image_type=`pcs_iscsi_image_type "$lun_dir"`
			local dev=`${image_type}_image_get_dev "$lun_dir"`
			${SCSI_TGT}_detach_dev dev=$dev target=$target lun_id=${lun#lun} 
			${image_type}_image_umount "$lun_dir"
		done
	fi

	rm -f "$ISCSI_ROOT/$target/control/.running"
	put_id $ctl_port $target
	return 0
}

function pcs_iscsi_down_lun {
	local target="$1"
	local lun_id="$2"
	local ctl_port="$3"

	if [ -z "$ctl_port" ] ; then
		echo "Unknown control port" 1>&2
		return 1
	fi

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 2

	${SCSI_TGT}_detach_lun target=$target lun_id=$lun_id ctl_port=$ctl_port
	if [ $? -ne 0 ] ; then
		echo "Can't delete LUN $lun_id for target $target" 1>&2
		return 3
	fi

	if [ -f "$ISCSI_ROOT/$target/lun${lun_id}/ploop" ]; then
		$PLOOP umount "$ISCSI_ROOT/$target/lun${lun_id}/ploop" >/dev/null
		ret=$?
		if [ $ret -ne 0 ] ; then
			echo "Failed to umount ploop image $ISCSI_ROOT/$target/lun${lun_id}/ploop" 1>&2
			return 4
		fi
	fi

	return 0
}

function pcs_iscsi_check_portals {
	local target="$1"
	local ctl_port="$2"

	[ ! -f "$ISCSI_ROOT/$target/control/address" ] && return 0
	addresses=`cat "$ISCSI_ROOT/$target/control/address" 2>/dev/null`
	for addr in `cat "$ISCSI_ROOT/$target/control/address" 2>/dev/null`; do
		local portal="$(echo "$addr" | cut -d '/' -f1):$ISCSI_PORT";
		${SCSI_TGT}_show_target_portals target=$target ctl_port=$ctl_port | grep "$portal" 2>/dev/null
		[ $? -ne 0 ] && return 1
	done
	return 0
}

function pcs_iscsi_get_ctl_port {
	local target="$1"
	cat "$ISCSI_SHM/.running/$target" 2>/dev/null
	return $?
}

function pcs_iscsi_get_status {
	local target="$1"

	if [ ! -f "$ISCSI_SHM/.running/$target" ] ; then
		echo "stopped"
		return 0
	fi

	ctl_port=`cat "$ISCSI_SHM/.running/$target" 2>/dev/null`
	if [ -z "$ctl_port" ] ; then
		echo "service failed"
		return 1
	fi

	pcs_iscsi_set_env $target
	if [ $? -ne 0 ] ; then
		echo "service failed"
		return 2
	fi

	${SCSI_TGT}_get_target_status target=$target ctl_port=$ctl_port
	return $?
}


# cleanup resources used by target
function pcs_iscsi_cleanup_target {

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 1

	if [ ! -f "$ISCSI_SHM/.running/$target" ] ; then
		echo "Target $target not running, nothing to cleanup" 1>&2
		return 0
	fi

	if [ -f "$ISCSI_ROOT/$target/control/address" ] ; then
		addresses=`cat "$ISCSI_ROOT/$target/control/address" 2>/dev/null`
		if [ -n "$addresses" ] ; then
			echo "Cleanup addresses for $target - $addresses" 1>&2
			${SCSI_TRANSPORT}_remove_addr $addresses
		fi
	fi
	luns=`ls -d "$ISCSI_ROOT/$target"/lun[0-9]* 2>/dev/null`
	if [ -n "$luns" ] ; then
		for l in $luns; do
			local lun=$(basename $l)
			local lun_dir="$ISCSI_ROOT/$target/$(basename $l)"
			local image_type=`pcs_iscsi_image_type "$lun_dir"`
			echo "Umount $lun_dir"
			${image_type}_image_umount "$lun_dir"
		done
	fi

	ctl_port=`cat $ISCSI_SHM/.running/$target 2>/dev/null`
	[ -n "$ctl_port" ] && put_id $ctl_port
	rm -f "$ISCSI_SHM/.running/$target"
	return 0
}

# start existing and registered target
function pcs_iscsi_up_target {
	local target="$1"
	local luns=""
	local ctl_port=""
	local addresses=""
	if [ ! -d  "$ISCSI_ROOT/$target" ] ; then
		echo "Target $target doesn't exist on $ISCSI_ROOT" 1>&2
		return 1
	fi

	pcs_iscsi_is_registered_local "$target"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "Target $target not registered on current host" 1>&2
		return 2
	fi

	[ ! -d "$ISCSI_SHM/.running" ] && mkdir -p "$ISCSI_SHM/.running"

	if [ -f "$ISCSI_SHM/.running/$target" ]; then
		echo "Target already running." 1>&2
		return 3
	fi

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 4

	if [ -f "$ISCSI_ROOT/$target/control/address" ] ; then
		addresses=`cat "$ISCSI_ROOT/$target/control/address"`
		if [ -z "$addresses" ] ; then
			echo "$target: no addresses found in \"$ISCSI_ROOT/$target/control/address\"" 1>&2
			return 5
		fi
		for addr in $addresses; do
			${SCSI_TRANSPORT}_add_addr $addr
			[ $? -ne 0 ] && return 5
		done
	else
		echo "$target: can't find file \"$ISCSI_ROOT/$target/control/address\"" 1>&2
		return 6
	fi

	ctl_port=`get_id`

	${SCSI_TGT}_create_target target=$target addresses="$addresses" ctl_port=$ctl_port
	if [ $? -ne 0 ] ; then
		echo "Unable add target $target" 1>&2
		${SCSI_TRANSPORT}_remove_addr $addresses
		pcs_iscsi_down_target $target $ctl_port
		return 7
	fi

	# FIXME: Add IP wildcard to allow all initiators (this is insecure!).
	# FIXME: We must read allowed initiators from $ISCSI_ROOT/$target/control/allowed
	${SCSI_TGT}_disable_authentication target=$target ctl_port=$ctl_port
	if [ $? -ne 0 ]; then
		echo "Failed to bind allowed initiators" 1>&2
		pcs_iscsi_down_target $target $ctl_port
		return 8
	fi

	pcs_iscsi_bind_account $target $ctl_port
	if [ $? -ne 0 ] ; then
		echo "Failed to bind CHAP account to target" 1>&2
		pcs_iscsi_down_target $target $ctl_port
		return 9
	fi

	${SCSI_TGT}_configure_target target=$target ctl_port=$ctl_port
	if [ $? -ne 0 ]; then
		echo "Failed to configure target" 1>&2
		pcs_iscsi_down_target $target $ctl_port
		return 10
	fi

	luns=`ls -d "$ISCSI_ROOT/$target/"lun[0-9]* 2>/dev/null`
	if [ -n "$luns" ] ; then
		for l in $luns; do
			pcs_iscsi_up_lun $target $ctl_port `basename $l`
			if [ $? -ne 0 ] ; then
				pcs_iscsi_down_target $target $ctl_port
				return 11
			fi
		done
	fi

	echo $ctl_port | echo_fsync "$ISCSI_ROOT/$target/control/.running"
	if [ $? -ne 0 ] ; then
		echo "Can't store file $ISCSI_ROOT/$target/control/.running" 1>&2
		pcs_iscsi_down_target $target $ctl_port
		return 12
	fi

	${SCSI_TGT}_enable_target target=$target ctl_port=$ctl_port
	if [ $? -ne 0 ] ; then
		echo "Can't enable target $target" 1>&2
		pcs_iscsi_down_target $target $ctl_port
		return 13
	fi

	echo $ctl_port > $ISCSI_SHM/.running/$target
	return 0
}

# delete existing target, force flag allow delete target registered
# on another host
function pcs_iscsi_delete_target {
	local target="$1"
	local force="$2"

	if [ -z "$ISCSI_ROOT" ] ; then
		echo "ISCSI_ROOT must be specified" 1>&2
		return 1
	fi

	[ ! -d "$ISCSI_ROOT/$target" ] && return 0

	pcs_iscsi_is_registered_local "$target"
	rc=$?
	[ $rc -gt 1 ] && return $rc # pcs_iscsi_is_registered_local failed
	if [ $rc -eq 0 ]; then
		# target registered on current host
		if [ -f "$ISCSI_SHM/.running/$target" ] ; then
			echo "Can't delete running target" 1>&2
			return 2
		fi
		pcs_iscsi_unregister_target "$target"
	elif [ -z "$force" ]; then
		pcs_iscsi_target_has_owner $target
		rc=$?
		[ $rc -gt 1 ] && return $rc

		if [ $rc -eq 1 ] ; then
			echo "Target $target registered on another host" 1>&2
			return 3
		fi
	fi

	pcs_iscsi_set_chap_account $target "-" "$ISCSI_ROOT/$target"
	if [ $? -ne 0 ] ; then
		return 5
	fi

	mv "$ISCSI_ROOT/$target" "$ISCSI_ROOT/tmp/$target"
	if [ $? -ne 0 ] ; then
		echo "Can't move directory $ISCSI_ROOT/$target to $ISCSI_ROOT/tmp/$target" 1>&2
		return 4
	fi

	# detach luns from deleted target
	luns=`ls -d "$ISCSI_ROOT/tmp/$target"/lun[0-9]* 2>/dev/null`
	if [ -n "$luns" ] ; then
		for l in $luns; do
			rm "$l/control/.attached" 2>/dev/null
		done
	fi

	rm -rf "$ISCSI_ETC/targets/$target" $ISCSI_ROOT/tmp/$target
	return 0
}

function pcs_iscsi_set_chap_account {
	local target="$1"
	local user_name="$2"
	local dir="$3"

	[ "$user_name" == "-" ] && user_name=""

	account_file="$dir/control/account"

	if [ -f $account_file ] ; then
		old_user=$(cat $account_file)
		$VSTORAGE_TARGET_BIN account-tag del -user $old_user -tag $target
		# retcode 12 is "(user) does not exist" - strange but ok
		if [ $? -ne 0 -a $? -ne 12 ] ; then
			echo "Failed to disassociate CHAP account $old_user with target $target" 1>&2
			return 1
		fi
		rm -f $account_file
	fi

	if [ -n "$user_name" ]; then
		$VSTORAGE_TARGET_BIN account-tag add -user $user_name -tag $target
		if [ $? -ne 0 ] ; then
			echo "Failed to set CHAP account $user_name. $target has no associated user now (was: $old_user)" 1>&2
			return 1
		fi
		echo $user_name | echo_fsync $account_file
		if [ $? -ne 0 ] ; then
			echo "Failed to write $account_file. $target has no associated user now (was: $old_user)" 1>&2
			$VSTORAGE_TARGET_BIN account-del add -user $user_name -tag $target
			return 1
		fi
	fi

	return 0
}

# create and register target
function pcs_iscsi_make_target {
	local target="$1"
	local addr="$2"
	local user_name="$3"
	local stor_attr=""

	if [ -z "$ISCSI_ROOT" ] ; then
		echo "ISCSI_ROOT must be specified" 1>&2
		return 1
	fi

	if [ -d "$ISCSI_ROOT/$target" ] ; then
		echo "Target $target already exist in $ISCSI_ROOT" 1>&2
		return 2
	fi

	tmp=`mktemp -d ${ISCSI_ROOT}/tmp/new-$target.XXXXXXXX`
	if [ $? -ne 0 ] ; then
		echo "Can't create temporary directory." 1>&2
		return 3
	fi

	is_root_on_vstorage || stor_attr="mds-storage=1"

	for dir in "$tmp/control" "$tmp/pr"; do
		mkdir -p "$dir" >/dev/null
		if [ $? -ne 0 ] ; then
			echo "Can't create directory $dir" 1>&2
			rm -rf $tmp
			return 4
		fi

		if [ -n "$stor_attr" ]; then
			pcs_iscsi_set_storage_attr "$dir" "$stor_attr"
			if [ $? -ne 0 ]; then
				rm -rf $tmp
				return 4
			fi
		fi
	done

	echo "$addr" | echo_fsync "$tmp/control/address"
	if [ $? -ne 0 ] ; then
		echo "Can't store address in file $tmp/control/address" 1>&2
		rm -rf $tmp
		return 5
	fi

	echo "$SCSI_TGT $SCSI_TRANSPORT" | echo_fsync "$tmp/control/metadata"
	if [ $? -ne 0 ] ; then
		echo "Can't store metadata in file $tmp/control/metadata" 1>&2
		rm -rf $tmp
		return 6
	fi

	if [ ! -z "$user_name" ]; then
		pcs_iscsi_set_chap_account "$target" "$user_name" "$tmp"
		if [ $? -ne 0 ]; then
			rm -rf $tmp
			return 7
		fi
	fi

	mv "$tmp" "${ISCSI_ROOT}/$target" >/dev/null
	if [ $? -ne 0 ] ; then
		echo "Failed to create directory." 1>&2
		rm -rf $tmp
		return 8
	fi

	pcs_iscsi_register_target $target
	if [ $? -ne 0 ] ; then
		echo "Unable register target $target on this host" 1>&2
		return 9
	fi

	return 0
}

function pcs_iscsi_check_root {
	local create_dir="$1"
	if [ -z "$ISCSI_ROOT" ] ; then
		echo "Unable to find directory with iSCSI targets, please check 'ISCSI_ROOT'" 1>&2
		echo "in $ISCSI_ETC/config" 1>&2
		exit 1
	fi

	if [ ! -d "$ISCSI_ROOT" ]; then
		if [ "$create_dir" != "yes" ] ; then
			echo "Directory $ISCSI_ROOT does not exist" 1>&2
			exit 2
		fi
		mkdir -p $ISCSI_ROOT
		if [ $? -ne 0 ] ; then
			echo "Unable create directory $ISCSI_ROOT" 1>&2
			exit 3
		fi
	fi

	[ ! -d "$ISCSI_ROOT/tmp" ] && mkdir -p "$ISCSI_ROOT/tmp"
}

function pcs_iscsi_check_target {
	local target="$1"
	if [ ! -d  "$ISCSI_ROOT/$target" ] ; then
		echo "Target $target doesn't exist on $ISCSI_ROOT" 1>&2
		exit 1
	fi

	pcs_iscsi_is_registered_local "$target"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "Target $target not registered on current host" 1>&2
		exit 2
	fi
}

function pcs_iscsi_target_can_stop_nonforcibly {
	local ctl_port="$1"; shift
	local target="$1"

	if [ -z "$ctl_port" ]; then
		echo "Control port is unknown" 1>&2
		return 1
	fi

	local initiators=$(${SCSI_TGT}_show_initiators target=$target ctl_port=$ctl_port)
	if [ -n "$initiators" ]; then
		echo "initiators still connected" 1>&2
		echo "$initiators" 1>&2
		return 2
	fi
	return 0
}

function pcs_iscsi_lock_exec {
	local cmd="$1" ; shift
	local target="$1"

	[ ! -d $ISCSI_SHM ] && mkdir -p $ISCSI_SHM
	(
		flock -w 30 -x 222
		if [ $? -ne 0 ] ; then
			echo "Unable lock target $target" 1>&2
			return 10;
		fi
		# '222<&-' explicitely closes fd 222 for childs, especially tgtd
		(${cmd} "${@}") 222<&-
		return $?
	) 222>"$ISCSI_SHM/.lock_$target"
	return $?
}

function pcs_iscsi_save_snapshot_info {
	local target="$1"
	local lun_id="$2"
	local uuid="$3"
	local descr="$4"
	mkdir -p "$ISCSI_ROOT/snapshots" >/dev/null 2>&1
	echo "SNAP_CREATE_DATE='$(date +"%F %T")'; SNAP_DESCRIPTION='${descr}'; SNAP_IQN='$target'; SNAP_LUN='${lun_id}'" \
		| echo_fsync "$ISCSI_ROOT/snapshots/$uuid"
	if [ $? -ne 0 ] ; then
		echo "Unable save description for new snapshot" 1>&2
		return 3
	fi
	return 0
}

function pcs_iscsi_image_type() {
	local lun_dir="$1"

	test -f $lun_dir/DiskDescriptor.xml && { echo ploop; return; }
	test -f $lun_dir/disk.img && { echo file; return; }
	echo "Unknown image format: $lun_dir" 1>&2
	exit 1
}

function ploop_delete_snapshot_info() {
	$PLOOP snapshot-list -H -o UUID "$lun_dir/DiskDescriptor.xml" | \
		while read snap_uuid; do
			snap_uuid=`expr "$snap_uuid" : "^\{\(.*\)\}$"`
			rm -f "$ISCSI_ROOT/snapshots/$snap_uuid"
		done
}

function pcs_iscsi_delete_snapshot_info() {
	local lun_dir="$1"
	img_type=`pcs_iscsi_image_type $lun_dir`

	case "$img_type" in
		ploop) ploop_delete_snapshot_info "$lun_dir"
		;;
	esac
}

function pcs_iscsi_get_mnt_root() {
	# ask shaman for ISCSI root
	cluster=`$SHAMAN get-config-local 2>/dev/null | grep -Po "CLUSTER_NAME=(\K\S+)"`
	[ -z "$cluster" ] && return
	grep "[pv]storage://$cluster" /proc/mounts | head -n1 | awk '{ print $2 }'
}

function pcs_iscsi_get_root() {
	local root="$1"
	if [ -n "$root" ] ; then
		echo "$root"
		return
	fi

	root="$ISCSI_ROOT"
	if [ -z "$root" ] ; then
		mnt=$(pcs_iscsi_get_mnt_root)
		[ -z "$mnt" ] && return
		root="$mnt/iscsi"
	fi
	echo "$root"
}

function pcs_iscsi_get_vols_root() {

	vols_root=$(grep -v "^#" /etc/vstorage/iscsi/config.json 2>/dev/null | sed -e 's/[{" \t}]/''/g' | awk -v RS=',' -F : '/VolumesRoot/ { print $2 }')
	if [ -z "$vols_root" ] ; then
			echo "Can't read VolumesRoot from /etc/vstorage/iscsi/config.json" 1>&2
			exit 1
	fi

	mnt="$(pcs_iscsi_get_mnt_root)"
	if [ -z "$mnt" ] ; then
		echo "Can't find vstorage mountpoint" 1>&2
		exit 1
	fi
	vols_root="$mnt/$vols_root"

	if [ ! -d "$vols_root" ] ; then
			echo "Directory $vols_root not found" 1>&2
			exit 1
	fi
	echo "$vols_root"
}

function pcs_iscsi_get_serial() {
        local target="$1"
        local lun="$2"
        local serial_sn=""

        serial_sn=$(grep -v "^#" $ISCSI_ROOT/$target/$lun/control/info.json 2>/dev/null | sed -e 's/[{" \t}]/''/g' | awk -v RS=',' -F : '/Serial/ { print $2 }')
        echo "$serial_sn"
}

function pcs_iscsi_set_limit {
	local target="$1"
	local type="$2"
	local limit="$3"
	local ctl_port=`pcs_iscsi_get_ctl_port $target`
	local limits_file="$ISCSI_ROOT/$target/control/limits"

	pcs_iscsi_set_env $target
	[ $? -ne 0 ] && return 1

	if [ "$(pcs_iscsi_get_status "$target")" == "running" ]; then
		${SCSI_TGT}_set_limit target=$target ctl_port=$ctl_port type=$type limit=$limit
		if [ $? -ne 0 ]; then
			echo "Unable to set limit=$limit" 1>&2
			return 2
		fi
	fi

	grep -q "$type" "$limits_file" 2>/dev/null && sed -i "s/$type=[0-9]*/$type=$limit/g" "$limits_file" || echo "$type=$limit" >> "$limits_file"
}

function pcs_iscsi_set_limit_from_file {
	local target="$1"
	local ctl_port=`pcs_iscsi_get_ctl_port $target`
	local limits_file="$ISCSI_ROOT/$target/control/limits"

	if [ $(pcs_iscsi_get_status "$target") != "running" ]; then
		echo "Target $target is not running" 1>&2
		return 1
	fi

	local kbps_limit=$(pcs_iscsi_get_limit "$target" "max_kbps")
	local iops_limit=$(pcs_iscsi_get_limit "$target" "max_iops")

	pcs_iscsi_set_limit "$target" "max_kbps" "$kbps_limit"
	pcs_iscsi_set_limit "$target" "max_iops" "$iops_limit"
}

function pcs_iscsi_get_limit {
	local target="$1"
	local type="$2"
	local limits_file="$ISCSI_ROOT/$target/control/limits"

	if [ ! -f $limits_file ]; then
		echo 0
		return 0
	fi

	local limit=$(grep -Po "(?<=$type=).*" "$limits_file" 2>&1)
	if [ -z "$limit" ]; then
		echo "Failed to read file \"$limits_file\"" 1>&2
		echo 0
		return 1
	fi
	echo "$limit"
}

function pcs_iscsi_is_lun_online {
	local target="$1"
	local lun_id="$2"
	local ctl_port=""

	ctl_port=`pcs_iscsi_get_ctl_port $target`
	if [ -z "$ctl_port" ] ; then
		echo "No"
		return 1
	fi

	pcs_iscsi_set_env $target
	if [ $? -ne 0 ] ; then
		echo "No"
		return 2
	fi

	online=`${SCSI_TGT}_is_lun_online target=$target lun_id=$lun_id ctl_port=$ctl_port`
	echo $online
	return 0
}

function pcs_iscsi_get_lun_stats {
	local target="$1"
	local lun_id="$2"
	local ctl_port=""

	ctl_port=`pcs_iscsi_get_ctl_port $target`
	if [ -z "$ctl_port" ] ; then
		echo ""
		return 1
	fi

	pcs_iscsi_set_env $target
	if [ $? -ne 0 ] ; then
		echo ""
		return 2
	fi

	${SCSI_TGT}_show_lun_stats target=$target lun_id=$lun_id ctl_port=$ctl_port
	return 0
}

function pcs_iscsi_show_initiators {
	local target="$target"
	local ctl_port=""

	ctl_port=`pcs_iscsi_get_ctl_port $target`
	if [ -z "$ctl_port" ] ; then
		echo ""
		return 1
	fi

	pcs_iscsi_set_env $target
	if [ $? -ne 0 ] ; then
		echo ""
		return 2
	fi

	initiators=`${SCSI_TGT}_show_initiators target=$target ctl_port=$ctl_port`
	if [ $? -ne 0 ] ; then
		echo ""
		return 3
	fi

	echo $initiators
	return 0
}
