#!/bin/bash
NFS_SHM="/dev/shm/vstorage-nfs"
VSTORAGE_ETC="/etc/vstorage"
GANESHA_ETC="/etc/ganesha"
NFS_ETC="$VSTORAGE_ETC/nfs"
BRCTL="/usr/sbin/brctl"
ARPING="/usr/sbin/arping"
SHAMAN="/usr/sbin/shaman"
DBUS_SEND="/usr/bin/dbus-send"
IP_CMD="/sbin/ip"
OVSCTL_CMD="/usr/bin/ovs-vsctl"
VSTORAGE_BIN="/usr/bin/vstorage"
KTUTIL_BIN="/usr/bin/ktutil"
KLIST_BIN="/usr/bin/klist"
PCS_OSTOR_VAR_RUN="/var/run/ostor"
NFS_VAR_RUN="$PCS_OSTOR_VAR_RUN/nfs"
NFS_V4_RECOV_ROOT="/var/lib/nfs/ganesha"
NFS_ENVIRONMENT="/run/sysconfig/ganesha"
NFS_GANESHA_PID="/var/run/ganesha.pid"
NFS_V4_RECOV_DIR="v4recov"
NFS_KEYTAB="/etc/krb5.keytab"
BRIDGE_NAME="vstorage-nfs"
NO_OWNER="unregistered"


nfs_get_netdev()
{
# Get a list of interfaces, excluding ones with LOOPBACK NOARP or SLAVE flags
# Accept mask: eth*[0-9]|tr*[0-9]|wlan[0-9]|ath[0-9]|ip6tnl*[0-9]|mip6mnha*[0-9]|bond[0-9]
        NETDEVICES=`${IP_CMD} a l | awk '
/^[[:digit:]]+:/ && /\<UP\>/ && !/\<LOOPBACK\>/ && !/\<SLAVE\>/ && !/\<NOARP\>/{
        if ($2 ~ /^en.*|^eth.*[0-9]+|^virbr[0-9]+|^em.*[0-9]+|^p[0-9]+p[0-9]+|^tr.*[0-9]+|^wlan[0-9]+|^ath[0-9]+|^ip6tnl.*[0-9]+|^mip6mnha.*[0-9]+|^vlan.*|^bond.*[0-9]+|^ib[0-9]+/) print $2;
}' | sed -e 's/:$//' -e 's/@.*//'`

        local tmp_list=""
        # Open vSwitch mode - IP assigned to bridge over device
        if [ -d "/sys/module/openvswitch" ]; then
                for dev in $NETDEVICES; do
                        local tmp
                        tmp=`$OVSCTL_CMD iface-to-br $dev 2>/dev/null`
                        [ $? -ne 0 ] && tmp="$dev"
                        tmp_list="$tmp_list $tmp"
                done
                NETDEVICES="$tmp_list"
        fi

        # step 2 - filter out devices without IP address
        tmp_list=""
        for dev in $NETDEVICES; do
                local tmp=`${IP_CMD} a l $dev | awk -v dev=$dev \
                        '/^^[\ \t]+inet6? / { print(dev); exit; }'`
                [ -n "$tmp" ] && tmp_list="$tmp_list $tmp"
        done
        echo "$tmp_list"
}

pcs_ha_validate_addr() {
        tmp_addr="$1"
        out=""
        for addr in $tmp_addr; do
                # validate addresses
                addr=`echo $addr | cut -d '/' -f 1`
                /usr/bin/ipcalc -c $addr
                if [ $? -ne 0 ] ; then
                        echo "Address '$addr' is wrong." 1>&2
                        return
                fi
                /usr/bin/ipcalc -c -6 $addr >/dev/null 2>&1
                if [ $? -ne 0 ] ; then
                        /bin/ipcalc -c -4 $addr >/dev/null 2>&1
                        if [ $? -ne 0 ] ; then
                                echo "Address '$addr' is wrong." 1>&2
                                return
                        fi
                        out="$addr/32 $out"
                else
                        out="$addr/64 $out"
                fi
        done
        echo "$out"
}


pcs_nfs_get_root() {
        local root="$1"
        if [ -n "$root" ] ; then
                echo "$root"
                return
        fi

        echo "$NFS_ROOT"
}


pcs_nfs_check_root() {
        local create_dir="$1"
        if [ -z "$NFS_ROOT" ] ; then
                echo "Unable to find directory with NFS shares, please check 'NFS_ROOT'" 1>&2
                echo "in $NFS_ETC/config, or use -r,--root option." 1>&2
		# return 111 error code to check it in UI agent and set NFS root to config.
                exit 111
        fi

        if [ ! -d "$NFS_ROOT" ]; then
                if [ "$create_dir" != "yes" ] ; then
                        echo "Directory $NFS_ROOT does not exist" 1>&2
                        exit 2
                fi
                mkdir -p $NFS_ROOT
                if [ ! -d $NFS_ROOT ] ; then
                        echo "Unable create directory $NFS_ROOT" 1>&2
                        exit 3
                fi
        fi

        [ ! -d "$NFS_ROOT/tmp" ] && mkdir -p "$NFS_ROOT/tmp"
	[ ! -d "$NFS_ROOT/shares" ] && mkdir -p "$NFS_ROOT/shares"
}


pcs_ha_lock_exec() {
        local cmd="$1" ; shift
        local share="$1"
        [ ! -d $NFS_SHM ] && mkdir -p $NFS_SHM
        (
                flock -w 30 -x 222
                if [ $? -ne 0 ] ; then
                        echo "Unable lock export $share" 1>&2
                        return 10;
                fi
                # '222<&-' explicitely closes fd 222 for childs, especially tgtd
                (${cmd} "${@}") 222<&-
                return $?
        ) 222>"$NFS_SHM/.lock_$share"
        return $?
}

pcs_ha_update_arp() {
        local addr="$1"
        if [ ! -x $ARPING ]; then
                echo "Can't find executable $ARPING" 1>&2
                return 1
        fi

       	net_dev=`nfs_get_netdev 2>/dev/null`
	if [ -z "$net_dev" ]; then
		echo "No active network interfaces" 1>&2
		return 2
	fi

	for dev in $net_dev; do
		$ARPING -c 10 -w 1 -q -U -I $dev $addr
	        if [ $? -ne 0 ] ; then
        	        echo "Unable send update ARP request" 1>&2
       	        	return 3
        	fi
        done

        return 0
}

pcs_ha_add_addr() {
        local addr="$1"
        local bridge_exist=`$BRCTL show | grep "$BRIDGE_NAME"`
        if [ -z "$bridge_exist" ] ; then
                echo "Bridge $BRIDGE_NAME doesn't exist on this host" 1>&2
                return 1
        fi
        addr_exist=`${IP_CMD} addr show $BRIDGE_NAME 2>/dev/null | grep "$addr"`
        if [ -n "$addr_exist" ] ; then
		echo "Address $addr already exist on bridge $BRIDGE_NAME" 1>&2
	else
        	$IP_CMD addr add "$addr" dev "$BRIDGE_NAME"
        	if [ $? -ne 0 ] ; then
                	echo "Failed to add address $addr to bridge $BRIDGE_NAME" 1>&2
                	return 2
        	fi
	fi

        pcs_ha_update_arp `echo $addr | cut -d '/' -f 1`
        [ $? -ne 0 ] && return 3
        return 0
}

pcs_ha_remove_addr() {
        while [ "${#}" -gt 0 ]; do
                local addr="$1"; shift
                local addr_exist=`${IP_CMD} addr show $BRIDGE_NAME 2>/dev/null | grep "$addr"`
                [ -n "$addr_exist" ] && $IP_CMD addr del "$addr" dev "$BRIDGE_NAME" >/dev/null 2>&1
        done
}

get_host_id() {
        cat $VSTORAGE_ETC/host_id
        rc=$?
        if [ $? -ne 0 ] ; then
                echo "Can't read host id" 1>&2
                return $rc
        fi
        return 0
}

pcs_ha_is_registered_local() {
        local target="$1"
        local host_id="$2"
        local owner=""
        if [ -z "$host_id" ] ; then
                host_id=`get_host_id`
                [ $? -ne 0 ] && return 2
        fi

        owner=`cat "$NFS_ROOT/shares/$target/control/host"`
        if [ $? -ne 0 ] ; then
                echo "Unable read file $target/control/host" 1>&2
                return 3
        fi

        [ -z "$owner" -o "$owner" != "$host_id" ] && return 1

        return 0
}

pcs_nfs_check_ganesha() {
	local service=""
        local timeout=30

        # Wait until ganesha register itself in DBUS
        while [ $timeout -gt 0 ] ; do
                service=$($DBUS_SEND --system --dest=org.freedesktop.DBus --type=method_call --print-reply /org/freedesktop/DBus \
        	org.freedesktop.DBus.ListNames 2>&1 | grep org.ganesha.nfsd)
		[ -n "$service" ] && return
		timeout=$[ $timeout - 1 ]
		sleep 1
	done

	echo "Ganesha's DBus service isn't registered" 1>&2
	exit 1
}

pcs_nfs_check_share() {
        local name="$1"
	if [ -z "$name" ] ; then
		echo "Name of share is empty" 1>&2
		exit 1
	fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "Share $name doesn't exist on $NFS_ROOT" 1>&2
                exit 1
        fi

	pcs_ha_is_registered_local "$name"
        rc=$?
        if [ $rc -ne 0 ] ; then
                [ $rc -eq 1 ] && echo "Share $name does not registered on current host" 1>&2
                exit 2
        fi
}

pcs_nfs_check_vol_id()
{
	local tmp_vol_id="$1"

	ls $NFS_ROOT/shares 2>/dev/null | while read share
	do
		vol_id=`cat $NFS_ROOT/shares/$share/control/volumeid`
		[ "$vol_id" == "$tmp_vol_id" ] && exit 1
		exit 0
	done

	return $?
}

pcs_nfs_list_registered() {

        host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

        ls "$NFS_ROOT/shares" 2>/dev/null | while read share
        do
                [ ! -d  "$NFS_ROOT/shares/$share/control" ] && continue

		pcs_ha_is_registered_local $share $host_id >/dev/null 2>&1
                [ $? -ne 0 ] && continue
                echo ""
                echo "nfs-$share"
                echo "$NFS_ROOT/$share"
        done
}

pcs_ha_init_host() {
        [ ! -d $NFS_SHM ] && mkdir -p $NFS_SHM/.running >/dev/null 2>&1
        # make bridge
        bridge_exist=`$BRCTL show | grep $BRIDGE_NAME`
        if [ -z "$bridge_exist" ] ; then
                $BRCTL addbr $BRIDGE_NAME
                if [ $? -ne 0 ] ; then
                        echo "Unable create bridge $BRIDGE_NAME" 1>&2
                        return 1
                fi
        fi

        return 0
}

pcs_nfs_has_owner() {
        local share="$1"
        local owner=""
	local hostfile="$NFS_ROOT/shares/$share/control/host"
        if [ -f "$hostfile" ]; then
                owner=`cat $hostfile`
                if [ $? -ne 0 ] ; then
                        echo "Unable read file $hostfile" 1>&2
                        return 3
                fi
        fi

        [ -n "$owner" -a "$owner" != "$NO_OWNER" ] && return 1
        return 0
}

pcs_nfs_unregister_ha() {

        local name="$1"
	msg=`echo "$NO_OWNER" | dd of="$NFS_ROOT/shares/$name/control/host" conv=fsync 2>&1`
	if [ $? -ne 0 ]; then
		echo "$msg" 1>&2
		echo "Can't write in file $tmp" 1>&2
		return 1
	fi

	return 0
}

pcs_nfs_register_ha() {
        local name="$1"
        local host_id
        host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

        msg=`echo $host_id | /bin/dd of="$NFS_ROOT/shares/$name/control/host" conv=fsync 2>&1`
        if [ $? -ne 0 ]; then
                echo "$msg" 1>&2
                echo "Unable save host_id in $NFS_ROOT/shares/$name/control/host" 1>&2
                return 2
        fi

        # reset running state after registration
	rm -f "$NFS_ROOT/shares/$name/control/.running"
        return 0
}

pcs_nfs_unregister() {
        local name="$1"
        host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

        prev_owner=`cat "$NFS_ROOT/shares/$name/control/host" 2>/dev/null`
	pcs_nfs_unregister_ha "$name"
        [ $? -ne 0 ] && return 2

        [ "$prev_owner" = "$host_id" ] && $SHAMAN del "nfs-$name" >/dev/null 2>&1
        return 0
}

function pcs_nfs_filter_shaman_exit_code {
        # exit code 5 means "node is not registered in the HA cluster"
        [ $1 -ne 0 -a $1 -ne 5 ] && return $1
        return 0
}

pcs_nfs_register() {
        local name="$1"
        local prev_owner=""
	hostfile="$NFS_ROOT/shares/$name/control/host"

        if [ -f "$hostfile" ]; then
                prev_owner=`cat "$hostfile"`
                if [ $? -ne 0 ] ; then
                        echo "Can't read $hostfile" 1>&2
                        return 1
                fi
        fi

	pcs_nfs_register_ha "$name"
        [ $? -ne 0 ] && return $?

        if [ -z "$prev_owner" -o "$prev_owner" = "$NO_OWNER" ]; then
                $SHAMAN add "nfs-$name" -P "$NFS_ROOT/shares/$name" >/dev/null
                pcs_nfs_filter_shaman_exit_code $?
                if [ $? -ne 0 ] ; then
                        echo "Can't register NFS share '$name' as shaman's resource" 1>&2
                        echo "Leave NFS share in unregistered state." 1>&2
			pcs_nfs_unregister "$name" >/dev/null 2>&1
                        return 2
                fi
        else
                $SHAMAN move-from "nfs-$name" "$prev_owner" >/dev/null
                pcs_nfs_filter_shaman_exit_code $?
                if [ $? -ne 0 ] ; then
			echo "Can't register NFS share '$name' as shaman's resource" 1>&2
			echo "Leave NFS share in unregistered state." 1>&2
			pcs_nfs_unregister "$name" >/dev/null 2>&1
                        return 3
                fi
        fi

        return 0
}

pcs_nfs_bind_service()
{
	local share_name="$1"
	local timeout=5
	local ret=0

	host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

	fsmds_id=`cat $NFS_ROOT/shares/$share_name/control/fsmds`
	if [ -z $fsmds_id ]; then
		echo "Can't get fsmds id from '$NFS_ROOT/shares/$share_name/control/fsmds'" 1>&2
		return 2
	fi	

	# ostor-agent can doesn't have new configuration yet. Try to bind several times with timeout
	while [ $timeout -gt 0 ] ; do
		ostor-ctl bind -H $host_id -S $fsmds_id >/dev/null
		ret=$?
		[ $ret -eq 0 ] && break
		timeout=$[ $timeout - 1 ]
		sleep 1
	done

	if [ $ret -ne 0 ] ; then
		echo "Can't bind: 'ostor-ctl bind -H $host_id -S $fsmds_id'" 1>&2
		return 3
	fi

	return 0
}

pcs_nfs_bind_state()
{
	local nfs_state="$NFS_ROOT/state"
	local gsh_recov="$NFS_V4_RECOV_ROOT/$NFS_V4_RECOV_DIR/"

	[ ! -d "$gsh_recov" ] && mkdir -p "$gsh_recov"

	stor_attr="mds-storage=1"
	if [ ! -d "$nfs_state" ]; then
		mkdir -p "$nfs_state"
		if [ $? -ne 0 ] ; then
			echo "Can't create state directory $nfs_state" 1>&2
			exit 1
		fi

		pcs_nfs_set_storage_attr "$nfs_state" "$stor_attr"
		if [ $? -ne 0 ]; then
			echo "Can't set storage attributes $nfs_state $stor_attr" 1>&2
			exit 2
		fi
	fi

	if ! mountpoint -q "$gsh_recov"; then
		mount --bind "$nfs_state" "$gsh_recov" 1>&2
		if [ $? -ne 0 ] ; then
			echo "Can't mount $nfs_state to $gsh_recov" 1>&2
			exit 3
		fi	
	fi
}

pcs_nfs_unbind_state()
{
	local gsh_recov="$NFS_V4_RECOV_ROOT/$NFS_V4_RECOV_DIR/$addr"
	
	umount $gsh_recov 1>&2
	if [ $? -ne 0 ] ; then
		echo "Can't umount $gsh_recov" 1>&2
		exit 1
	fi
}

pcs_nfs_start_grace()
{
	local addr="$1"

	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace_ip string:"$addr" >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to start grace period for $addr" 1>&2
		return 1
	fi

	return 0
}

pcs_nfs_check_keytab()
{
	keytab="$1"
	# NFS share keytab should contain only one nfs principal
	count=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep -ioP ".*" | uniq  | wc -l)
	if [ $count -ne 1 ]; then
		echo "Keytab should contain one principal for NFS server" 1>&2
		return 1
	fi

	count=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep "nfs/" | uniq | wc -l)
	if [ $count -ne 1 ]; then
		echo "Invalid name of principal for NFS server. Should be nfs/hostname@REALM" 1>&2
		return 2
	fi

	return 0
}

pcs_nfs_get_principal()
{
	local keytab="$1"
	local principal=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep "nfs/" | uniq)
	echo "$principal"
}

pcs_nfs_del_principal()
{
	local keytab="$1"
	local entry="$2"
	local tmp_keytab=`mktemp --dry-run`

	[ ! -f $keytab ] && return 0

	if [ -z $entry ]; then
		echo "Unable to delete empty entry from keytab" 1>&2
		return 2
	fi

	# write keytab to tmp file
	$KTUTIL_BIN < <(echo -e "rkt $keytab \n wkt $tmp_keytab \n") > /dev/null 2>&1

	# delete all matched entry from keytab
	while true; do
		[ ! -f $tmp_keytab ] && break;
		# find the first occurrence entry
		slot=$($KTUTIL_BIN < <(echo -e "rkt $tmp_keytab \n list \n") | grep "$entry" | awk ' /.[1-9]/ {print $1} ' | head -1)
		[ -z $slot ] && break;
		# delete the entry and write the new tmp keytab
		local tmp_keytab2=`mktemp --dry-run`
		$KTUTIL_BIN < <(echo -e "rkt $tmp_keytab \n delent $slot \n wkt $tmp_keytab2 \n") > /dev/null 2>&1
		rm -f $tmp_keytab >/dev/null 2>&1
		tmp_keytab="$tmp_keytab2"
	done

	rm -f $keytab >/dev/null 2>&1
	[ -f $tmp_keytab ] && mv "$tmp_keytab" "$keytab"

	return $?
}

pcs_nfs_unmerge_keytab()
{
	local share="$1"
	local share_keytab="$NFS_ROOT/shares/$share/control/krb5.keytab"
	local principal=""

	if [[ -f "$share_keytab" && -f "$KTUTIL_BIN" ]]; then
		principal=`pcs_nfs_get_principal $share_keytab`
		[ -z $principal ] && return 0;
		pcs_nfs_del_principal $NFS_KEYTAB $principal
		return $?
	fi

	return 0
}

pcs_nfs_merge_keytab()
{
	local share="$1"
	local share_keytab="$NFS_ROOT/shares/$share/control/krb5.keytab"

	if [[ -f "$share_keytab" && -f "$KTUTIL_BIN" ]]; then
		$KTUTIL_BIN < <(echo -e "rkt $share_keytab \n wkt $NFS_KEYTAB \n") > /dev/null 2>&1;
		if [ $? -ne 0 ]; then
			echo "Can't merge keytab $share_keytab into $NFS_KEYTAB" 1>&2
			return 1
		fi
	fi

	return 0
}

pcs_nfs_write_share_cfg()
{
	local share="$1"

	addr=`cat "$NFS_ROOT/shares/$name/control/address" | cut -d '/' -f1`
        if [ -z "$addr" ] ; then
                echo "Unable to read address for share '$name'" 1>&2
                return 1
        fi

	volumeid=`cat "$NFS_ROOT/shares/$share/control/volumeid"`
        if [ -z "$volumeid" ] ; then
                echo "Unable to read volume id for share '$name'" 1>&2
                return 2
        fi

	[ ! -d "$NFS_VAR_RUN" ] && mkdir -p "$NFS_VAR_RUN"

	echo  $addr | dd of="$NFS_VAR_RUN/${volumeid}.cfg" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store file $NFS_VAR_RUN/${volumeid}.cfg" 1>&2
		return 1
	fi

}

pcs_nfs_delete_share_cfg()
{
	local share="$1"
	volumeid=`cat "$NFS_ROOT/shares/$share/control/volumeid"`
        if [ -z "$volumeid" ] ; then
                echo "Unable to read volume id for share '$name'" 1>&2
                return 1
        fi

	rm -f $NFS_VAR_RUN/${volumeid}.cfg >/dev/null 2>&1	
	return 0
}

pcs_nfs_set_storage_attr() 
{
	local dir="$1"; shift
	$VSTORAGE_BIN "set-attr" "$dir" ${@} >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Can't set storage attributes for $dir" 1>&2
		return 1
	fi
	return 0
}

pcs_nfs_start_ganesha()
{
	source $NFS_ENVIRONMENT
	${NUMACTL} ${NUMAOPTS} /usr/bin/ganesha.nfsd ${OPTIONS} ${EPOCH}
	if [ $? -ne 0 ]; then
		echo "Can't start nfs ganesha: (${NUMACTL} ${NUMAOPTS} /usr/bin/ganesha.nfsd ${OPTIONS} ${EPOCH})" 1>&2
		exit 1
	fi

	# Wait while ganesha initialize DBus connection
	sleep 3
}

pcs_nfs_stop_ganesha()
{
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to stop nfs ganesha" 1>&2
		exit 1
	fi
}

pcs_nfs_copy_ds_list()
{
	[ ! -f "$NFS_ROOT/ds.list" ] && return
        [ ! -d "$NFS_VAR_RUN" ] && mkdir -p "$NFS_VAR_RUN"
        
	cp "$NFS_ROOT/ds.list" "$NFS_VAR_RUN/ds.list" >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't copy $NFS_ROOT/ds.list to $NFS_VAR_RUN/ds.list"
		return 1
	fi
}

pcs_nfs_delete_ds_list()
{
	rm -f $NFS_VAR_RUN/ds.list >/dev/null 2>&1
	return $?
}

pcs_ganesha_is_exported() {
	local export_id="$1"
	
	display_export=$($DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.DisplayExport \
	uint16:${export_id}  2>&1 | grep 'uint16 $export_id')

	if [ -z $display_export ]; then
		return 0
	fi
	
	echo " Ganesha export with id $export_id exists" 1>&2
	return 1
}

pcs_ganesha_gen_export_id() {
	while true; do
		#gen export_id in range from 1001-65535
		export_id=$(/usr/bin/shuf -i 1001-65535 -n 1)
		echo "export_id=$export_id"
		#check if current export_id already exists
		pcs_ganesha_is_exported $export_id
		[ $? -eq 0 ] && break;
	done
}

pcs_ganesha_add_export() {
	local export_id="$1"
	local config_name="$2"
	local config="$GANESHA_ETC/ostorfs/$config_name"

	if [ ! -f $config ]; then
		echo "Unable find export config '$config'" 1>&2
		return 1
	fi

	#add export to ganesha
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
		string:$config string:"export(export_id=${export_id})" >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable add export to ganesha" 1>&2
		return 2
	fi

	return 0
}

pcs_ganesha_del_export() {	
	local export_id="$1"

	#delete export from ganesha
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.RemoveExport uint16:$export_id >/dev/null 2>&1
}

pcs_nfs_down_export() {
	local share="$1"
	local export_name="$2"
	local export_path="$NFS_ROOT/shares/$share/exports/$export_name"
	local export_config="${share}_${export_name}.conf"
	local export_id=

	if [ -f "$export_path/export_id" ]; then
		export_id=`cat $export_path/export_id`
		if [ -z "$export_id" ] ; then
			echo "Unable to read export id for export '$export_path'" 1>&2
			return 1
		fi
	else
		echo "Unable find ganesha export id in export '$export_path'" 1>&2
		return 2
	fi
	
	pcs_ganesha_del_export $export_id

	rm -f $GANESHA_ETC/ostorfs/$export_config >/dev/null 2>&1

	return $?
}

pcs_nfs_up_export() {
	local share="$1"
	local export_name="$2"
	local export_path="$NFS_ROOT/shares/$share/exports/$export_name"
	local export_config="${share}_${export_name}.conf"
	local export_id=

	# read export_id
	if [ -f "$export_path/export_id" ]; then
		export_id=`cat $export_path/export_id`
		if [ -z "$export_id" ] ; then
			echo "Unable to read export id for export '$export_path'" 1>&2
			return 1
		fi
	else
		echo "Unable find ganesha export id in export '$export_path'" 1>&2
		return 2
	fi

	# create local directory with configs
	if [ ! -d $GANESHA_ETC/ostorfs/ ]; then
		mkdir -p $GANESHA_ETC/ostorfs/ >/dev/null 2>&1
	fi

	#cp config from shared storage to local
	cp $export_path/$export_config $GANESHA_ETC/ostorfs/
	if [ $? -ne 0 ]; then
		echo "Unable to copy config '$export_path/$export_config' to '$GANESHA_ETC/ostorfs/'" 1>&2
		return 3
	fi

	pcs_ganesha_add_export $export_id $export_config
	if [ $? -ne 0 ]; then
		echo "Unable to add '$export_config' export to ganesha" 1>&2
		return 4
	fi

	return 0
}

pcs_nfs_parse_config()
{
	local tmp="$1"
	local share="$2"
	local export_config="$3"

	if [ ! -f "$export_config" ]; then
		echo "Unable find export config '$export_config'" 1>&2
		return 1
	fi

	# Config should have one export
	num_exports=`cat $export_config | grep -ioP "(?<=export_id).*" | wc -l`
	if [ $num_exports -ne 1 ]; then
		echo "Invalid number of exports in config '$export_config '. Export should be exaclty one per config" 1>&2
		return 2
	fi

	# Extract export id from config
	export_id=`cat $export_config | grep -ioP "(?<=export_id).*" | grep -ioP "\d+"`
	if [ -z "$export_id" ]; then
		echo "Can't extract export id from '$export_config'" 1>&2
		return 3
	fi

	# Save export_id
	pcs_ganesha_is_exported $export_id
	if [ $? -ne 0 ] ; then
		echo "Export with id $export_id already exported" 1>&2
		return 4
	fi
	echo  $export_id | dd of="$tmp/export_id" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store file $tmp/export_id" 1>&2
		return 5
	fi

	# Extract and check volumeid
	volumeid=`cat $export_config | grep -ioP "(?<=volumeid).*" | grep -ioP "\w*"`
	if [ -z "$volumeid" ]; then
		echo "Can't extract volume id from '$export_config'" 1>&2
		return 6
	fi
	share_volumeid=`cat "$NFS_ROOT/shares/$share/control/volumeid"`
	if [ "$share_volumeid" != "$volumeid" ]; then
		echo "Export volume id should be the same as share volumeid. '$share_volumeid' != '$volumeid' " 1>&2
		return 7
	fi

	return 0
}

# make export on shared storage
pcs_nfs_make_export() {
	local tmp="$1"
	local share="$2"
	local export_name="$3"
	local export_config="$4"

	if [ -z "$NFS_ROOT" ] ; then
		echo "NFS_ROOT must be specified" 1>&2
		return 1
	fi

	if [ ! -d "$NFS_ROOT/shares/$share/exports" ] ; then
		echo "Unable find directory $NFS_ROOT/shares/$share/shares" 1>&2
		return 2
	fi

	if [ -d "$NFS_ROOT/shares/$share/exports/$export_name" ]; then
		echo "Export '$export_name' already exist for share '$share'" 1>&2
		return 3
	fi

	# Set storage attributes 
	stor_attr="mds-storage=1"
	pcs_nfs_set_storage_attr "$tmp" "$stor_attr"
	if [ $? -ne 0 ]; then
		echo "Can't set storage attributes $tmp $stor_attr" 1>&2
		rm -rf $tmp
		return 4
	fi
	
	pcs_nfs_parse_config $tmp $share $export_config
	if [ $? -ne 0 ] ; then
		echo "Can't parse config $export_config"
		return 5
	fi
	
	#cp ganeshas config to export directory
	cp "$export_config" "$tmp/${share}_${export_name}.conf"
	if [ $? -ne 0 ] ; then
		echo "Can't copy config $export_config to $tmp/${share}_${export_name}.conf"
		return 6
	fi

	mv "$tmp" "$NFS_ROOT/shares/$share/exports/$export_name"
	if [ $? -ne 0 ] ; then
		echo "Can't rename $tmp to $NFS_ROOT/shares/$share/exports/$export_name" 1>&2
		rm -rf $tmp >/dev/null 2>&1
		return 7
	fi

	return 0
}

pcs_nfs_make_share() {
	local name="$1"
	local vol_id="$2"
	local addr="$3"

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

	if [ -d "$NFS_ROOT/shares/$name" ] ; then 
		echo "share '$name' already exist at $NFS_ROOT/shares" 1>&2
		return 2
	fi

	pcs_nfs_check_vol_id $vol_id
	if [ $? -ne 0 ] ; then
		echo "Share with the specified volume id($vol_id) already exist." 1>&2
		return 3
	fi

	tmp=`mktemp -d "$NFS_ROOT/tmp/new-$name.XXXXXXXX"`
	if [ $? -ne 0 ] ; then
		echo "Can't create temporary directory." 1>&2
		return 4
	fi

	# Create share directories and set storage attributes
	stor_attr="mds-storage=1"
	for dir in "$tmp/control" "$tmp/exports"; do
		mkdir -p "$dir" >/dev/null
		if [ $? -ne 0 ] ; then
			echo "Can't create directory $dir" 1>&2
			rm -rf $tmp
			return 10
		fi

		pcs_nfs_set_storage_attr "$dir" "$stor_attr"
		if [ $? -ne 0 ]; then
			echo "Can't set storage attributes $dir $stor_attr" 1>&2
			rm -rf $tmp
			return 11
		fi
	done

	# Save address
	echo "$addr" | dd of="$tmp/control/address" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store address in file $tmp/control/address" 1>&2
		rm -rf $tmp
		return 12
	fi

	# Save volume_id
	echo "$vol_id" | dd of="$tmp/control/volumeid" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store volumeid in file $tmp/control/volumeid" 1>&2
		rm -rf $tmp
		return 13
	fi

	# Save fsmds id
	fsmds_id=$(ostor-ctl get-config -S -i $vol_id | awk ' {print $1,$2} ' | grep FS | awk '{print $1}')
	if [ -z $fsmds_id ]; then
		echo "Can't get fsmds id for volume $vol_id" 1>&2
		return 14
	fi
	echo "$fsmds_id" | dd of="$tmp/control/fsmds" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store fsmds id($fsmds_id) in file $tmp/control/fsmds" 1>&2
		rm -rf $tmp
		return 15
	fi

	mv $tmp "$NFS_ROOT/shares/$name"
	rc=$?
	if [ $? -ne 0 ]; then
		echo "Can't move $tmp to $NFS_ROOT/shares/$name" 1>&2
		rm -rf $tmp
		return 16
	fi

	pcs_nfs_register "$name"
	if [ $? -ne 0 ] ; then
		echo "Unable to register new share on current host" 1>&2
		return 17
	fi

	pcs_nfs_bind_service "$name"
	if [ $? -ne 0 ] ; then
		echo "Unable to bind FS service" 1>&2
		return 18
	fi

	return 0
}

pcs_nfs_delete_share() {
	local name="$1"
	local force="$2"

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	[ $rc -gt 1 ] && return $rc
	if [ $rc -eq 0 ]; then
		# target registered on current host
		if [ -f "$NFS_SHM/.running/$name" ] ; then
			echo "Can't delete running share" 1>&2
			return 2
		fi
		pcs_nfs_unregister "$name"
	elif [ -z "$force" ]; then
		pcs_nfs_has_owner $name
		rc=$?
		[ $rc -gt 1 ] && return $rc

		if [ $rc -eq 1 ] ; then
			echo "share '$name' registered on another host" 1>&2
			return 3
		fi
	fi

	mv "$NFS_ROOT/shares/$name" "$NFS_ROOT/tmp/$name"
	if [ $? -ne 0 ] ; then
		echo "Can't move directory $NFS_ROOT/shares/$name to $NFS_ROOT/tmp/$name"
		return 4
	fi

	rm -rf $NFS_ROOT/tmp/$name

	return 0
}

pcs_nfs_stop_share() {
	local name="$1"
	local force="$2"
	local addr_ha=""
        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "The NFS share '$name' doesn't exist in $NFS_ROOT" 1>&2
                return 2
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "share '$name' does not registered on current host" 1>&2
		return $rc
	fi

	if [ -z "$force" ]; then
		if [ ! -f "$NFS_SHM/.running/$name" ]; then
			echo "The NFS share '$name' not running" 1>&2
			return 0
		fi
	fi

        if [ -f "$NFS_ROOT/shares/$name/control/address" ] ; then
                addr_ha=`cat "$NFS_ROOT/shares/$name/control/address"`
		pcs_ha_remove_addr $addr_ha
        fi

	#unmerge share keytab from global	
	pcs_nfs_unmerge_keytab $name

	pcs_nfs_delete_share_cfg $name
	
	rm -f "$NFS_ROOT/shares/$name/control/.running"
        exports=`ls "$NFS_ROOT/shares/$name/exports" 2>/dev/null`
        for s in $exports; do
                [ ! -d "$NFS_ROOT/shares/$name/exports/$s" ] && continue
                pcs_nfs_down_export "$name" "$s"
        done

	rm -f "$NFS_SHM/.running/$name"
	return 0
}

pcs_nfs_get_status() {
	local name="$1"
	if [ -z "$NFS_ROOT" -o ! -d  "$NFS_ROOT/shares/$name" ] ; then
		echo "unknown"
		return 1
	fi

	if [ ! -f "$NFS_SHM/.running/$name" ]; then
		echo "stopped"
		return 0
	fi

	echo "running"
	return 0
}

pcs_nfs_start_share() {
	local name="$1"
	local force="$2"
	local addr=""
        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "The NFS share '$name' doesn't exist in $NFS_ROOT" 1>&2
                return 2
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "Share '$name' does not registered on current host" 1>&2
		return $rc
	fi

	[ ! -d "$NFS_SHM/.running" ] && mkdir -p "$NFS_SHM/.running"
	if [ -z "$force" ]; then
        	if [ -f "$NFS_SHM/.running/$name" ]; then
                	echo "Share '$name' already running." 1>&2
                	return 3
	        fi
	fi

	addr_ha=`cat "$NFS_ROOT/shares/$name/control/address"`
        if [ -z "$addr_ha" ] ; then
                echo "Unable to read address for share '$name'" 1>&2
                return 4
        fi

	pcs_nfs_write_share_cfg $name
	if [ $? -ne 0 ] ; then 
                echo "Unable to write config for share '$name'" 1>&2
                return 5
        fi

	exports=`ls "$NFS_ROOT/shares/$name/exports" 2>/dev/null`
	for s in $exports; do
		[ ! -d "$NFS_ROOT/shares/$name/exports/$s" ] && continue
		pcs_nfs_up_export "$name" "$s"
		if [ $? -ne 0 ] ; then
			pcs_nfs_stop_share "$name" 1
			return 6
		fi
	done

	#merge share keytab into global	
	pcs_nfs_merge_keytab $name
	if [ $? -ne 0 ] ; then 
		pcs_nfs_stop_share "$name" 1
		return 7
	fi

	# enable grace period before adding IP
	addr=`echo $addr_ha | cut -d '/' -f1`
	pcs_nfs_start_grace "$addr"
	if [ $? -ne 0 ] ; then 
		pcs_nfs_stop_share "$name" 1
		return 8
	fi

	pcs_ha_add_addr $addr_ha
	if [ $? -ne 0 ] ; then 
		pcs_nfs_stop_share "$name" 1
		return 9
	fi

	echo "1" | dd of="$NFS_ROOT/shares/$name/control/.running" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store file $NFS_ROOT/shares/$name/control/.running" 1>&2
		pcs_nfs_stop_share "$name" 1
		return 10
	fi

        echo "1" > "$NFS_SHM/.running/$name"
	return 0
}

