#!/usr/bin/bash
NFS_SHM="/dev/shm/vstorage-nfs"
VSTORAGE_ETC="/etc/vstorage"
NFS_ETC="$VSTORAGE_ETC/nfs"
NFS_VAR_RUN="/run/vstorage/nfs"
PVFS_DS_LIST_PATH="$NFS_VAR_RUN/ds_list_v2"
GANESHA_ETC="/etc/ganesha"
GANESHA_EXPORTS_ETC="$GANESHA_ETC/vstorage-exports"
BRCTL="/usr/sbin/brctl"
ARPING="/usr/sbin/arping"
SHAMAN="/usr/sbin/shaman"
DISCOVCTL="/usr/bin/discovctl"
PVFS_DS_REGISTRY_KVSTOR_KEY="/vstorage/pvfs/ds"
DBUS_SEND="/usr/bin/dbus-send"
IP_CMD="/sbin/ip"
OVSCTL_CMD="/usr/bin/ovs-vsctl"
VSTORAGE_BIN="/usr/bin/vstorage"
KTUTIL_BIN="/usr/bin/ktutil"
KLIST_BIN="/usr/bin/klist"
NFS_V4_RECOV_ROOT="/var/lib/nfs/ganesha"
LOGFILE="/var/log/ganesha/ganesha.log"
CONFFILE="/etc/ganesha/ganesha.conf"
NFS_ENVIRONMENT="/run/sysconfig/ganesha"
NFS_GANESHA_PID="/run/ganesha.pid"
NFS_V4_RECOV_DIR="v4recov"
NFS_KEYTAB="/etc/krb5.keytab"
BRIDGE_NAME="vstorage-nfs"
NO_OWNER="unregistered"
NFS_LIB_VERSION=2
PLOOP_CMD="/usr/sbin/ploop"
MPFS_VAR_RUN="/run/vstorage/mpfs"
MPFS_FSAL_CONFIG_PATH="$MPFS_VAR_RUN/mpfs_fsal.config"
MPFS_DS_CMD="/usr/bin/vstorage-mpfs-ds"
MPFS_DS_CONFIG="/etc/vstorage/mpfs/ds.config"
MPFS_DS_PID_FILE="/run/vstorage-mpfs-ds.pid"
MPFS_DS_VAR_RUN="$MPFS_VAR_RUN/ds"
MPFS_MDS_CMD="/usr/bin/vstorage-mpfs-mds"
MPFS_MDS_CONFIG="/etc/vstorage/mpfs/mds.config"
MPFS_MDS_PID_FILE="/run/vstorage-mpfs-mds.pid"
MPFS_MDS_VAR_RUN="$MPFS_VAR_RUN/mds"
PVFS_VDISKS_MOUNT_DIR="/mnt/vdisks/vstorage/pvfs"
MPFS_DS_VDISKS_MOUNT_DIR="/mnt/vdisks/vstorage/mpfs/ds"
MPFS_MDS_VDISKS_MOUNT_DIR="/mnt/vdisks/vstorage/mpfs/mds"

# Legacy variables, for compatibility with the OSTOR FSAL
PCS_OSTOR_VAR_RUN="/run/ostor"
OSTOR_NFS_VAR_RUN="$PCS_OSTOR_VAR_RUN/nfs"

load_matching_nfs_lib_version() {
	local share=$1
	local path="$NFS_ROOT/shares/$share/control/version"
	local version=""

	if [ -f "$path" ] ; then
		version=`cat "$path"`
		if [ $? -ne 0 ] ; then
			echo "Unable to read file $path" 1>&2
			exit 1
		fi
	fi

	switch_to_nfs_lib_version "$version"
}

switch_to_nfs_lib_version() {
	local desired_version=$1

	if [ "x$desired_version" != "x$NFS_LIB_VERSION" ] ; then
		NFS_LIB_VERSION=""
		source "${BASH_SOURCE[0]%%[0-9]*}$desired_version"
	fi
}

pcs_nfs_parse_size() {
	/usr/bin/numfmt --from=auto "$1"
}

get_swanctl_path() {
    local conn_id=$1
    echo "/etc/swanctl/conf.d/nfs-${conn_id}.conf"
}

add_ipsec_bypass() {
    local addr_n_prefix=$1
    local addr=${addr_n_prefix/\/*/}
    local conn_id=${addr//[.:]/-}

    cat > $(get_swanctl_path $conn_id) <<EOF && systemctl reload-or-try-restart strongswan.service
    connections {
        bypass-nfs {
            children {
                ${conn_id} {
                    local_ts = ${addr}[tcp/2049]
                    mode = pass
                    start_action = trap
                }
            }
        }
    }
EOF
}

del_ipsec_bypass() {
    local addr_n_prefix=$1
    local addr=${addr_n_prefix/\/*/}
    local conn_id=${addr//[.:]/-}

    rm -f $(get_swanctl_path $conn_id) &&
    systemctl reload-or-try-restart strongswan.service
}

nfs_get_netdev()
{
# Get a list of interfaces, excluding ones with LOOPBACK NOARP or SLAVE flags
# Accept mask: eth*[0-9]|tr*[0-9]|wlan[0-9]|ath[0-9]|ip6tnl*[0-9]|mip6mnha*[0-9]|bond[0-9]
        NETDEVICES=`${IP_CMD} a l | awk '
/^[[:digit:]]+:/ && /\<UP\>/ && !/\<LOOPBACK\>/ && !/\<SLAVE\>/ && !/\<NOARP\>/{
        if ($2 ~ /^en.*|^eth.*[0-9]+|^virbr[0-9]+|^em.*[0-9]+|^p[0-9]+p[0-9]+|^tr.*[0-9]+|^wlan[0-9]+|^ath[0-9]+|^ip6tnl.*[0-9]+|^mip6mnha.*[0-9]+|^vlan.*|^bond.*[0-9]+|^ib[0-9]+/) print $2;
}' | sed -e 's/:$//' -e 's/@.*//'`

        local tmp_list=""
        # Open vSwitch mode - IP assigned to bridge over device
        if [ -d "/sys/module/openvswitch" ]; then
                for dev in $NETDEVICES; do
                        local tmp
                        tmp=`$OVSCTL_CMD iface-to-br $dev 2>/dev/null`
                        [ $? -ne 0 ] && tmp="$dev"
                        tmp_list="$tmp_list $tmp"
                done
                NETDEVICES="$tmp_list"
        fi

        # step 2 - filter out devices without IP address
        tmp_list=""
        for dev in $NETDEVICES; do
                local tmp=`${IP_CMD} a l $dev | awk -v dev=$dev \
                        '/^^[\ \t]+inet6? / { print(dev); exit; }'`
                [ -n "$tmp" ] && tmp_list="$tmp_list $tmp"
        done
        echo "$tmp_list"
}

pcs_ha_validate_addr() {
        tmp_addr="$1"
        out=""
        for addr in $tmp_addr; do
                # validate addresses
                addr=`echo $addr | cut -d '/' -f 1`
                /usr/bin/ipcalc -c $addr
                if [ $? -ne 0 ] ; then
                        echo "Address '$addr' is wrong." 1>&2
                        return
                fi
                /usr/bin/ipcalc -c -6 $addr >/dev/null 2>&1
                if [ $? -ne 0 ] ; then
                        /bin/ipcalc -c -4 $addr >/dev/null 2>&1
                        if [ $? -ne 0 ] ; then
                                echo "Address '$addr' is wrong." 1>&2
                                return
                        fi
                        out="$addr/32 $out"
                else
                        out="$addr/64 $out"
                fi
        done
        echo "$out"
}


pcs_nfs_get_root() {
        local root="$1"
        if [ -n "$root" ] ; then
                echo "$root"
                return
        fi

        echo "$NFS_ROOT"
}


pcs_nfs_check_root() {
        local create_dir="$1"
        if [ -z "$NFS_ROOT" ] ; then
                echo "Unable to find directory with NFS shares, please check 'NFS_ROOT'" 1>&2
                echo "in $NFS_ETC/config, or use -r,--root option." 1>&2
		# return 111 error code to check it in UI agent and set NFS root to config.
                exit 111
        fi

        if [ ! -d "$NFS_ROOT" ]; then
                if [ "$create_dir" != "yes" ] ; then
                        echo "Directory $NFS_ROOT does not exist" 1>&2
                        exit 2
                fi
                mkdir -p $NFS_ROOT
                if [ ! -d $NFS_ROOT ] ; then
                        echo "Unable create directory $NFS_ROOT" 1>&2
                        exit 3
                fi
        fi

        [ ! -d "$NFS_ROOT/tmp" ] && mkdir -p "$NFS_ROOT/tmp"
	[ ! -d "$NFS_ROOT/shares" ] && mkdir -p "$NFS_ROOT/shares"
}


pcs_ha_lock_exec() {
        local cmd="$1" ; shift
        local share="$1"
        [ ! -d $NFS_SHM ] && mkdir -p $NFS_SHM
        (
                flock -w 30 -x 222
                if [ $? -ne 0 ] ; then
                        echo "Unable lock export $share" 1>&2
                        return 10;
                fi
                # '222<&-' explicitely closes fd 222 for childs, especially tgtd
                (${cmd} "${@}") 222<&-
                return $?
        ) 222>"$NFS_SHM/.lock_$share"
        return $?
}

pcs_ha_update_arp() {
        local addr="$1"
        if [ ! -x $ARPING ]; then
                echo "Can't find executable $ARPING" 1>&2
                return 1
        fi

       	net_dev=`nfs_get_netdev 2>/dev/null`
	if [ -z "$net_dev" ]; then
		echo "No active network interfaces" 1>&2
		return 2
	fi

	for dev in $net_dev; do
		$ARPING -c 10 -w 1 -q -U -I $dev $addr
	        if [ $? -ne 0 ] ; then
        	        echo "Unable send update ARP request" 1>&2
       	        	return 3
        	fi
        done

        return 0
}

pcs_ha_add_addr() {
        local addr="$1"
        local bridge_exist=`$BRCTL show | grep "$BRIDGE_NAME"`
        if [ -z "$bridge_exist" ] ; then
                echo "Bridge $BRIDGE_NAME doesn't exist on this host" 1>&2
                return 1
        fi
        addr_exist=`${IP_CMD} addr show $BRIDGE_NAME 2>/dev/null | grep "$addr"`
        if [ -n "$addr_exist" ] ; then
		echo "Address $addr already exist on bridge $BRIDGE_NAME" 1>&2
	else
        	$IP_CMD addr add "$addr" dev "$BRIDGE_NAME"
        	if [ $? -ne 0 ] ; then
                	echo "Failed to add address $addr to bridge $BRIDGE_NAME" 1>&2
                	return 2
        	fi
	fi

        pcs_ha_update_arp `echo $addr | cut -d '/' -f 1`
        [ $? -ne 0 ] && return 3

        add_ipsec_bypass "$addr" || return 4

        return 0
}

pcs_ha_remove_addr() {
        while [ "${#}" -gt 0 ]; do
                local addr="$1"; shift
                local addr_exist=`${IP_CMD} addr show $BRIDGE_NAME 2>/dev/null | grep "$addr"`
                [ -n "$addr_exist" ] && $IP_CMD addr del "$addr" dev "$BRIDGE_NAME" >/dev/null 2>&1
                del_ipsec_bypass "$addr"
        done
}

get_host_id() {
        local host_id
	read -r host_id _ < "$VSTORAGE_ETC/host_id"
	local rc=$?
        if [ $rc -ne 0 ] ; then
                echo "Can't read $VSTORAGE_ETC/host_id, error $rc" 1>&2
                return 1
        fi
	if [ -z "$host_id" ]; then
		echo "$VSTORAGE_ETC/host_id is empty" 1>&2
		return 2
	fi
	echo "$host_id"
        return 0
}

pcs_ha_is_registered_local() {
        local target="$1"
        local host_id="$2"
        local owner=""
        if [ -z "$host_id" ] ; then
                host_id=`get_host_id`
                [ $? -ne 0 ] && return 2
        fi

        owner=`cat "$NFS_ROOT/shares/$target/control/host"`
        if [ $? -ne 0 ] ; then
                echo "Unable read file $target/control/host" 1>&2
                return 3
        fi

        [ -z "$owner" -o "$owner" != "$host_id" ] && return 1

        return 0
}

pcs_nfs_check_ganesha() {
	local service=""
        local timeout=30

        # Wait until ganesha register itself in DBUS
        while [ $timeout -gt 0 ] ; do
                service=$($DBUS_SEND --system --dest=org.freedesktop.DBus --type=method_call --print-reply /org/freedesktop/DBus \
        	org.freedesktop.DBus.ListNames 2>&1 | grep org.ganesha.nfsd)
		[ -n "$service" ] && return
		timeout=$[ $timeout - 1 ]
		sleep 1
	done

	echo "Ganesha's DBus service isn't registered" 1>&2
	exit 1
}

pcs_nfs_wait_until_ganesha_stopped() {
	local pid=$1
	if [ -z "$pid" ]; then
		pid=$(cat $NFS_GANESHA_PID)
	fi

	if [ -z "$pid" ]; then
		echo "Cannot load PID from $NFS_GANESHA_PID," \
			"service is not running?" 1>&2
		return 1
	fi

	while kill -0 $pid 2>/dev/null; do
		sleep 1
	done
}

pcs_nfs_check_share() {
        local name="$1"
	if [ -z "$name" ] ; then
		echo "Name of share is empty" 1>&2
		exit 1
	fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "Share $name doesn't exist on $NFS_ROOT" 1>&2
                exit 1
        fi

	pcs_ha_is_registered_local "$name"
        rc=$?
        if [ $rc -ne 0 ] ; then
                [ $rc -eq 1 ] && echo "Share $name is not registered on current host" 1>&2
                exit 2
        fi
}

pcs_nfs_check_vol_id()
{
	local tmp_vol_id="$1"

	ls $NFS_ROOT/shares 2>/dev/null | while read share
	do
		vol_id=`cat $NFS_ROOT/shares/$share/control/volumeid`
		[ "$vol_id" == "$tmp_vol_id" ] && exit 1
		exit 0
	done

	return $?
}

pcs_ha_init_host() {
        [ ! -d $NFS_SHM ] && mkdir -p $NFS_SHM/.running >/dev/null 2>&1
        # make bridge
        bridge_exist=`$BRCTL show | grep $BRIDGE_NAME`
        if [ -z "$bridge_exist" ] ; then
                $BRCTL addbr $BRIDGE_NAME
                if [ $? -ne 0 ] ; then
                        echo "Unable create bridge $BRIDGE_NAME" 1>&2
                        return 1
                fi
        fi

        return 0
}

pcs_ha_fence_host() {
        bridge_exist=`$BRCTL show | grep $BRIDGE_NAME`
        [ -z "$bridge_exist" ] && return 0
        # destroy bridge to delete all its IPs at once
        $BRCTL delbr $BRIDGE_NAME
        if [ $? -ne 0 ] ; then
                echo "failed to delete bridge $BRIDGE_NAME" 1>&2
		return 1
        fi
	return 0
}

pcs_nfs_has_owner() {
        local share="$1"
        local owner=""
	local hostfile="$NFS_ROOT/shares/$share/control/host"
        if [ -f "$hostfile" ]; then
                owner=`cat $hostfile`
                if [ $? -ne 0 ] ; then
                        echo "Unable read file $hostfile" 1>&2
                        return 3
                fi
        fi

        [ -n "$owner" -a "$owner" != "$NO_OWNER" ] && return 1
        return 0
}

pcs_nfs_unregister_ha() {
        local name="$1"
	msg=`echo "$NO_OWNER" | dd of="$NFS_ROOT/shares/$name/control/host" conv=fsync 2>&1`
	if [ $? -ne 0 ]; then
		echo "$msg" 1>&2
		echo "Can't write in file $tmp" 1>&2
		return 1
	fi

	return 0
}

pcs_nfs_register_ha() {
        local name="$1"
        local host_id
        host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

        msg=`echo $host_id | /bin/dd of="$NFS_ROOT/shares/$name/control/host" conv=fsync 2>&1`
        if [ $? -ne 0 ]; then
                echo "$msg" 1>&2
                echo "Unable save host_id in $NFS_ROOT/shares/$name/control/host" 1>&2
                return 2
        fi

        # reset running state after registration
	rm -f "$NFS_ROOT/shares/$name/control/.running"
        return 0
}

pcs_nfs_unregister() {
        local name="$1"
        host_id=`get_host_id`
        [ $? -ne 0 ] && return 1

        prev_owner=`cat "$NFS_ROOT/shares/$name/control/host" 2>/dev/null`
	pcs_nfs_unregister_ha "$name"
        [ $? -ne 0 ] && return 2

        [ "$prev_owner" = "$host_id" ] && $SHAMAN del "nfs-$name" >/dev/null 2>&1
        return 0
}

function pcs_nfs_filter_shaman_exit_code {
        # exit code 5 means "node is not registered in the HA cluster"
        [ $1 -ne 0 -a $1 -ne 5 ] && return $1
        return 0
}

pcs_nfs_register() {
        local name="$1"
        local prev_owner=""
	hostfile="$NFS_ROOT/shares/$name/control/host"

        if [ -f "$hostfile" ]; then
                prev_owner=`cat "$hostfile"`
                if [ $? -ne 0 ] ; then
                        echo "Can't read $hostfile" 1>&2
                        return 1
                fi
        fi

	pcs_nfs_register_ha "$name"
        [ $? -ne 0 ] && return $?

        if [ -z "$prev_owner" -o "$prev_owner" = "$NO_OWNER" ]; then
                $SHAMAN add "nfs-$name" -P "$NFS_ROOT/shares/$name" >/dev/null
                pcs_nfs_filter_shaman_exit_code $?
                if [ $? -ne 0 ] ; then
                        echo "Can't register NFS share '$name' as shaman's resource" 1>&2
                        echo "Leave NFS share in unregistered state." 1>&2
			pcs_nfs_unregister "$name" >/dev/null 2>&1
                        return 2
                fi
        else
                $SHAMAN move-from "nfs-$name" "$prev_owner" >/dev/null
                pcs_nfs_filter_shaman_exit_code $?
                if [ $? -ne 0 ] ; then
			echo "Can't register NFS share '$name' as shaman's resource" 1>&2
			echo "Leave NFS share in unregistered state." 1>&2
			pcs_nfs_unregister "$name" >/dev/null 2>&1
                        return 3
                fi
        fi

        return 0
}

pcs_nfs_bind_service()
{
	# Unused in this version
	return 0
}

pcs_nfs_bind_state()
{
	local nfs_state="$NFS_ROOT/state"
	local gsh_recov="$NFS_V4_RECOV_ROOT/$NFS_V4_RECOV_DIR/"

	[ ! -d "$gsh_recov" ] && mkdir -p "$gsh_recov"

	stor_attr="mds-storage=1"
	if [ ! -d "$nfs_state" ]; then
		mkdir -p "$nfs_state"
		if [ $? -ne 0 ] ; then
			echo "Can't create state directory $nfs_state" 1>&2
			exit 1
		fi

		pcs_nfs_set_storage_attr "$nfs_state" "$stor_attr"
		if [ $? -ne 0 ]; then
			echo "Can't set storage attributes $nfs_state $stor_attr" 1>&2
			exit 2
		fi
	fi

	if ! mountpoint -q "$gsh_recov"; then
		mount --bind "$nfs_state" "$gsh_recov" 1>&2
		if [ $? -ne 0 ] ; then
			echo "Can't mount $nfs_state to $gsh_recov" 1>&2
			exit 3
		fi
	fi
}

pcs_nfs_unbind_state()
{
	local gsh_recov="$NFS_V4_RECOV_ROOT/$NFS_V4_RECOV_DIR/$addr"

	umount $gsh_recov 1>&2
	if [ $? -ne 0 ] ; then
		echo "Can't umount $gsh_recov" 1>&2
		exit 1
	fi
}

pcs_nfs_start_grace()
{
	local addr="$1"

	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace_ip string:"$addr" >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to start grace period for $addr" 1>&2
		return 1
	fi

	return 0
}

pcs_nfs_check_keytab()
{
	keytab="$1"
	# NFS share keytab should contain only one nfs principal
	count=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep -ioP ".*" | uniq  | wc -l)
	if [ $count -ne 1 ]; then
		echo "Keytab should contain one principal for NFS server" 1>&2
		return 1
	fi

	count=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep "nfs/" | uniq | wc -l)
	if [ $count -ne 1 ]; then
		echo "Invalid name of principal for NFS server. Should be nfs/hostname@REALM" 1>&2
		return 2
	fi

	return 0
}

pcs_nfs_get_principal()
{
	local keytab="$1"
	local principal=$($KLIST_BIN -k -t "$keytab" | awk ' {print $4} ' | grep "nfs/" | uniq)
	echo "$principal"
}

pcs_nfs_del_principal()
{
	local keytab="$1"
	local entry="$2"
	local tmp_keytab=`mktemp --dry-run`

	[ ! -f $keytab ] && return 0

	if [ -z $entry ]; then
		echo "Unable to delete empty entry from keytab" 1>&2
		return 2
	fi

	# write keytab to tmp file
	$KTUTIL_BIN < <(echo -e "rkt $keytab \n wkt $tmp_keytab \n") > /dev/null 2>&1

	# delete all matched entry from keytab
	while true; do
		[ ! -f $tmp_keytab ] && break;
		# find the first occurrence entry
		slot=$($KTUTIL_BIN < <(echo -e "rkt $tmp_keytab \n list \n") | grep "$entry" | awk ' /.[1-9]/ {print $1} ' | head -1)
		[ -z $slot ] && break;
		# delete the entry and write the new tmp keytab
		local tmp_keytab2=`mktemp --dry-run`
		$KTUTIL_BIN < <(echo -e "rkt $tmp_keytab \n delent $slot \n wkt $tmp_keytab2 \n") > /dev/null 2>&1
		rm -f $tmp_keytab >/dev/null 2>&1
		tmp_keytab="$tmp_keytab2"
	done

	rm -f $keytab >/dev/null 2>&1
	[ -f $tmp_keytab ] && mv "$tmp_keytab" "$keytab"

	return $?
}

pcs_nfs_unmerge_keytab()
{
	local share="$1"
	local share_keytab="$NFS_ROOT/shares/$share/control/krb5.keytab"
	local principal=""

	if [[ -f "$share_keytab" && -f "$KTUTIL_BIN" ]]; then
		principal=`pcs_nfs_get_principal $share_keytab`
		[ -z $principal ] && return 0;
		pcs_nfs_del_principal $NFS_KEYTAB $principal
		return $?
	fi

	return 0
}

pcs_nfs_merge_keytab()
{
	local share="$1"
	local share_keytab="$NFS_ROOT/shares/$share/control/krb5.keytab"

	if [[ -f "$share_keytab" && -f "$KTUTIL_BIN" ]]; then
		$KTUTIL_BIN < <(echo -e "rkt $share_keytab \n wkt $NFS_KEYTAB \n") > /dev/null 2>&1;
		if [ $? -ne 0 ]; then
			echo "Can't merge keytab $share_keytab into $NFS_KEYTAB" 1>&2
			return 1
		fi
	fi

	return 0
}

pcs_nfs_get_this_host_ip_addrs()
{
	local net_dev=`nfs_get_netdev 2>/dev/null`
	if [ -z "$net_dev" ]; then
		echo "No active network interfaces" 1>&2
		return 1
	fi

	local addr_list=()
	local addrs
	for dev in $net_dev; do
		addrs=$("$IP_CMD" -4 addr show "$dev" 2>/dev/null \
				| awk '/inet / {print $2}' | cut -d / -f 1)
		[ $? -ne 0 -o -z "$addrs" ] && continue

		addr_list+=($addrs)
	done

	echo "${addr_list[*]}"
	return 0
}

pcs_nfs_get_ds_public_ip_addrs()
{
	local this_host_id=`get_host_id`
	[ -z "$this_host_id" ] && return 1

	if [ -z "$NFS_ROOT" ] ; then
		echo "NFS_ROOT must be specified" 1>&2
		return 2
	fi

	while read line || [ -n "$line" ]; do
		# Extract host ID
		host_id=$(echo "$line" | rev | cut -d ' ' -f 1 | rev)

		if [ "$host_id" != "$this_host_id" ]; then
			continue
		fi

		# Extract IPs
		ip_addrs_str=$(echo "$line" | rev | cut -d ' ' -f 2- | rev)
		IFS=", " read -r -a ip_addrs_list <<< "$ip_addrs_str"
		echo "${ip_addrs_list[*]}"
		return 0
	done < "$NFS_ROOT/ds.list"

	return 3
}

# Get a value from a pcs_config_file-formatted file
pcs_get_config_value()
{
	local key="$1"
	local file="$2"

	if [ -z "$key" -o -z "$file" ]; then
		echo "Key and file must be specified" 1>&2
		return 1
	fi

	if [ ! -f "$file" ]; then
		echo "File $file doesn't exist" 1>&2
		return 2
	fi

	local value=$( \
		grep -E "^$key *=" "$file" | \
		head -n 1 | \
		awk -F'[[:space:]]*=[[:space:]]*' '{print $2}' \
		)
	if [ -z "$value" ]; then
		echo "Can't find value for key $key in $file" 1>&2
		return 3
	fi

	echo "$value"
	return 0
}

pcs_get_cluster_name_impl()
(
	set -o pipefail
	"$SHAMAN" stat | grep "Cluster" | head -n 1 | awk -F"'" '{print $2}'
)

pcs_get_cluster_name()
{
	if [ -z "$CLUSTER_NAME" ]; then
		CLUSTER_NAME=$(pcs_get_cluster_name_impl)
		local rc=$?
		if [ $rc -ne 0 ]; then
			echo "Can't get cluster name, error $rc" 1>&2
			return 1
		fi
	fi
	echo "$CLUSTER_NAME"
}

pcs_run_discovctl()
{
	local ns="$1" ; shift

	if [ -z "$ns" ]; then
		echo "Namespace must be specified" 1>&2
		return 1
	fi

	local cluster_name=$(pcs_get_cluster_name)
	if [ -z "$cluster_name" ]; then
		echo "Can't get cluster name" 1>&2
		return 2
	fi

	"$DISCOVCTL" --cluster "$cluster_name" --ns "$ns" "$@"
	local rc=$?
	if [ $rc -ne 0 ]; then
		local args=$(printf '"%s" ' "$@")
		echo "Exit code $rc from command:" \
			"\"$DISCOVCTL\" --cluster \"$cluster_name\"" \
			"--ns \"$ns\" ${args% }" 1>&2
		return 3
	fi

	return 0
}

pcs_find_service_instances()
{
	local ns="$1" ; shift

	local service_list
	service_list=$(set -o pipefail ; \
		pcs_run_discovctl "$ns" find "$@" | \
		grep -v "^#" || true)
	if [ $? -ne 0 ]; then
		return 1
	fi

	local service_ids=()
	while IFS=$'\t' read -r service_id _; do
		service_ids+=("$service_id")
	done <<< "$service_list"
	echo "${service_ids[*]}"

	return 0
}

pcs_register_service()
{
	local ns="$1" ; shift

	pcs_run_discovctl "$ns" register "$@"
}

pcs_unregister_service()
{
	local ns="$1"
	local service_id="$2"

	pcs_run_discovctl "$ns" unregister --id "$service_id"
}

pcs_unregister_services()
{
	local ns="$1" ; shift

	local service_ids
	service_ids=$(pcs_find_service_instances "$ns" "$@")
	if [ $? -ne 0 ]; then
		return 1
	fi

	local rc=0
	for service_id in $service_ids; do
		pcs_unregister_service "$ns" "$service_id"
		if [ $? -ne 0 ]; then
			rc=2
		fi
	done
	return $rc
}

pcs_register_this_host_as_ds()
{
	local kvstor_key="$1"

	[ -z "$kvstor_key" ] && return 1

	local host_id
	host_id=`get_host_id`
	[ $? -ne 0 ] && return 2

	local ip_addrs
	ip_addrs=`pcs_nfs_get_this_host_ip_addrs`
	if [ $? -ne 0 ] ; then
		pcs_unregister_host_as_ds "$host_id" "$kvstor_key"
		return 3
	fi

	local pub_ip_addrs
	pub_ip_addrs=`pcs_nfs_get_ds_public_ip_addrs`
	if [ $? -ne 0 ] ; then
		pcs_unregister_host_as_ds "$host_id" "$kvstor_key"
		return 4
	fi

	declare -A is_public
	for ip in $pub_ip_addrs; do
		is_public["$ip"]=1
	done

	local priv_ip_addrs=()
	for ip in $ip_addrs; do
		if [ -z "${is_public["$ip"]}" ]; then
			priv_ip_addrs+=("$ip")
		fi
	done
	priv_ip_addrs="${priv_ip_addrs[*]}"

	local value="$host_id:${pub_ip_addrs// /,}:${priv_ip_addrs// /,}"

	"$SHAMAN" kvtool put "$kvstor_key/$host_id" "$value" >/dev/null
}

pcs_unregister_host_as_ds()
{
	local host_id="$1"
	local kvstor_key="$2"

	[ -z "$host_id" ] && return 1
	[ -z "$kvstor_key" ] && return 1

	"$SHAMAN" kvtool del "$kvstor_key/$host_id" >/dev/null
}

pcs_unregister_this_host_as_ds()
{
	local kvstor_key="$1"

	local host_id=`get_host_id`
	[ -z "$host_id" ] && return 1

	pcs_unregister_host_as_ds "$host_id" "$kvstor_key"
}

pcs_list_ds_vdisks()
{
	local share="$1"

	local path="$NFS_ROOT/shares/$share/control/mds_vdisk_id"
	local mds_vdisk_id=$(<"$path")
	if [ -z "$mds_vdisk_id" ]; then
		echo "Can't read $path" 1>&2
		return 1
	fi

	path="$NFS_ROOT/shares/$share/vdisks"
	local list
	list=$(set -o pipefail ; ls -1 "$path" | \
		grep -v "$mds_vdisk_id" || true)
	if [ $? -ne 0 ]; then
		echo "Can't list $path" 1>&2
		return 2
	fi

	echo -n "$list"

	return 0
}

pcs_nfs_write_share_cfg()
{
	local share="$1"

	local address=`cat "$NFS_ROOT/shares/$share/control/address" \
		| cut -d '/' -f1`
	if [ -z "$address" ] ; then
		echo "Unable to read address for share '$share'" 1>&2
		return 1
	fi

	[ ! -d "$NFS_VAR_RUN" ] && mkdir -p "$NFS_VAR_RUN"

	local dst_path="$NFS_VAR_RUN/${share}.address"
	echo $address | dd of="$dst_path" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't write to $dst_path" 1>&2
		pcs_nfs_delete_share_cfg "$share"
		return 2
	fi

	for param in fs_id mds_vdisk_id; do
		local src_path="$NFS_ROOT/shares/$share/control/$param"
		local dst_path="$NFS_VAR_RUN/${share}.$param"
		dd if="$src_path" of="$dst_path" conv=fsync >/dev/null 2>&1
		if [ $? -ne 0 ] ; then
			echo "Can't copy $src_path to $dst_path" 1>&2
			pcs_nfs_delete_share_cfg "$share"
			return 3
		fi
	done

	# Write <share>.ds_vdisks
	local dst_path="$NFS_VAR_RUN/${share}.ds_vdisks"
	pcs_list_ds_vdisks $share >"$dst_path"
	if [ $? -ne 0 ] ; then
		echo "Can't write to $dst_path" 1>&2
		pcs_nfs_delete_share_cfg "$share"
		return 30
	fi
}

pcs_nfs_delete_share_cfg()
{
	local share="$1"

	for param in address fs_id mds_vdisk_id ds_vdisks; do
		rm -f "$NFS_VAR_RUN/${share}.$param" >/dev/null 2>&1
	done
	return 0
}

pcs_nfs_set_storage_attr()
{
	local dir="$1"; shift
	$VSTORAGE_BIN "set-attr" "$dir" ${@} >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Can't set storage attributes for $dir" 1>&2
		return 1
	fi
	return 0
}

pcs_pvfs_vdisk_generate_id() {
	local vdisk_id
	# NOTE: this algorithm doesn't guarantee that there will be
	# no conflicts, as it doesn't prevent the parallel generation
	# of identical IDs. However, the probability of such event
	# is presumed to be extremely low.
	while true; do
		vdisk_id=$(/usr/bin/md5sum /proc/sys/kernel/random/uuid \
				| head -c 16)
		[ $? -ne 0 ] && return 1
		pcs_pvfs_vdisk_exists $vdisk_id
		[ $? -eq 0 ] && break
	done
	echo -n $vdisk_id
	return 0
}

pcs_pvfs_vdisk_exists() {
	local vdisk_id="$1"
	local count=$( \
		ls -d -1 $NFS_ROOT/shares/*/vdisks/$vdisk_id 2>/dev/null \
		| wc -l \
	)
	return $count
}

pcs_pvfs_vdisk_create() {
	local share_path="$1"
	local vdisk_id="$2"
	local size="$3"
	local vdisk_path="$share_path/vdisks/$vdisk_id"
	local image_path="$vdisk_path/image"

	if [ -d "$vdisk_path" ]; then
		echo "Vdisk $vdisk_id already exists" 1>&2
		return 1
	fi

	for path in "$vdisk_path" "$image_path"; do
		mkdir "$path"
		if [ $? -ne 0 ] ; then
			echo "Can't create directory $path" 1>&2
			rm -rf "$vdisk_path"
			return 2
		fi
	done

	(cd "$image_path" && \
		"$PLOOP_CMD" init -s $(($size/512)) -f expanded -t ext4 data \
		>/dev/null)
	if [ $? -ne 0 ] ; then
		echo "Cannot create vDisk image at $image_path" >&2
		rm -rf "$vdisk_path"
		return 3
	fi

	return 0
}

pcs_pvfs_find_share_by_vdisk_id() {
	local vdisk_id="$1"
	local share

	ls "$NFS_ROOT/shares" 2>/dev/null | \
	while read share; do
		if [ -d "$NFS_ROOT/shares/$share/vdisks/$vdisk_id" ]; then
			echo "$share"
			break
		fi
	done
}

pcs_vdisk_mount() {
	local share="$1"
	local vdisk_id="$2"
	local mount_dir="$3"

	if [ -z "$mount_dir" ]; then
		echo "Mount directory must be specified" >&2
		return 1
	fi

	local mount_point="$mount_dir/$vdisk_id"
	local vdisk_path="$NFS_ROOT/shares/$share/vdisks/$vdisk_id"
	local image_path="$vdisk_path/image"

	if [ -e "$mount_point/.mounted" ]; then
		return 0
	fi

	mkdir -p "$mount_point"
	if [ $? -ne 0 ]; then
		echo "Cannot create $mount_point" >&2
		return 1
	fi

	if [ ! -d "$image_path" ]; then
		echo "Cannot find a vDisk image at $image_path" >&2
		return 2
	fi

	(cd "$image_path" && \
		"$PLOOP_CMD" mount -m "$mount_point" DiskDescriptor.xml \
		>/dev/null)
	if [ $? -ne 0 ]; then
		echo "Cannot mount $image_path at $mount_point" >&2
		return 3
	fi

	echo "1" >"$mount_point/.mounted"
	if [ $? -ne 0 ]; then
		echo "Cannot create $mount_point/.mounted" >&2
		(cd "$image_path" && \
			"$PLOOP_CMD" umount DiskDescriptor.xml >/dev/null)
		return 4
	fi

	return 0
}

pcs_vdisk_unmount() {
	local vdisk_id="$1"
	local mount_dir="$2"

	if [ -z "$mount_dir" ]; then
		echo "Mount directory must be specified" >&2
		return 1
	fi

	local mount_point="$mount_dir/$vdisk_id"

	if [ ! -e "$mount_point/.mounted" ]; then
		return 0
	fi

	"$PLOOP_CMD" umount -m "$mount_point" >/dev/null
	if [ $? -ne 0 ]; then
		echo "Cannot unmount $mount_point" >&2
		return 1
	fi

	rmdir "$mount_point"

	return 0
}

pcs_vdisk_unmount_all() {
	local mount_dir="$1"
	local vdisk_id
	local ret=0
	local ls_mounts

	ls_mounts=`ls "$mount_dir" 2>/dev/null`
	while read vdisk_id; do
		pcs_vdisk_unmount "$vdisk_id" "$mount_dir"
		if [ $? -ne 0 ] ; then
			ret=1
		fi
	done <<< "$ls_mounts"
	return $ret
}

pcs_nfs_create_dir_with_vstorage_attrs() {
	local dir="$1"; shift
	local vstorage_attrs=("$@")

	mkdir -p "$dir" >/dev/null
	if [ $? -ne 0 ] ; then
		echo "Can't create directory $dir" 1>&2
		return 1
	fi

	if [ ${#vstorage_attrs[@]} -ne 0 ] ; then
		pcs_nfs_set_storage_attr "$dir" "$vstorage_attrs[@]"
		if [ $? -ne 0 ]; then
			echo "Can't set storage attributes of $dir to $vstorage_attrs[*]" >&2
			return 2
		fi
	fi

	return 0
}

pcs_nfs_start_ganesha()
{
	source $NFS_ENVIRONMENT

	OPTIONS="-L $LOGFILE -f $CONFFILE -N NIV_EVENT -p $NFS_GANESHA_PID"

	${NUMACTL} ${NUMAOPTS} /usr/bin/ganesha.nfsd ${OPTIONS} ${EPOCH}
	if [ $? -ne 0 ]; then
		echo "Can't start nfs ganesha: (${NUMACTL} ${NUMAOPTS} /usr/bin/ganesha.nfsd ${OPTIONS} ${EPOCH})" 1>&2
		exit 1
	fi

	# Wait while ganesha initialize DBus connection
	sleep 3
}

pcs_nfs_stop_ganesha()
{
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to stop nfs ganesha" 1>&2
		exit 1
	fi
}

pcs_nfs_copy_ds_list()
{
	[ ! -f "$NFS_ROOT/ds.list" ] && return
        [ ! -d "$OSTOR_NFS_VAR_RUN" ] && mkdir -p "$OSTOR_NFS_VAR_RUN"

	cp "$NFS_ROOT/ds.list" "$OSTOR_NFS_VAR_RUN/ds.list" >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't copy $NFS_ROOT/ds.list to $OSTOR_NFS_VAR_RUN/ds.list"
		return 1
	fi
}

pcs_nfs_delete_ds_list()
{
	rm -f $OSTOR_NFS_VAR_RUN/ds.list >/dev/null 2>&1
	return $?
}

pcs_load_registered_ds_list()
{
	local kvstor_key="$1"
	[ -z "$kvstor_key" ] && return 1

	echo '# HOST_ID:PUB_IP_ADDR_LIST:PRIV_IP_ADDR_LIST'
	"$SHAMAN" kvtool get \
		--prefix $kvstor_key/ --values-only \
		| sort
}

pcs_create_scripts_link()
{
	local scripts_dir_name="$1"
	local link_path="$2"
	local scripts_dir="$(dirname ${BASH_SOURCE[0]})/$scripts_dir_name"

	ln -s "$scripts_dir" "$link_path"
	if [ $? -ne 0 ] ; then
		echo "Cannot create link $link_path to $scripts_dir" 1>&2
		return 1
	fi
	return 0
}

pcs_generate_mpfs_fsal_config()
{
	local cluster_name=$(pcs_get_cluster_name)
	if [ -z "$cluster_name" ]; then
		echo "Can't get cluster name" 1>&2
		return 1
	fi

	echo "cluster_name=$cluster_name"

	return 0
}

pvfs_fsal_configure()
{
	[ -n "$ENABLE_PVFS" ] || return 0

	pcs_register_this_host_as_ds "$PVFS_DS_REGISTRY_KVSTOR_KEY"
	if [ $? -ne 0 ]; then
		return 1
	fi

	mkdir -p "$NFS_VAR_RUN"
	if [ $? -ne 0 ] ; then
		echo "Cannot create $NFS_VAR_RUN" 1>&2
		return 2
	fi

	# Create links to scripts for PVFS FSAL module
	rm -f "$NFS_VAR_RUN/scripts"
	pcs_create_scripts_link "pvfs" "$NFS_VAR_RUN/scripts"
	if [ $? -ne 0 ]; then
		return 3
	fi

	# Write ds_list_v2 for PVFS FSAL module
	local ds_list
	ds_list=`pcs_load_registered_ds_list $PVFS_DS_REGISTRY_KVSTOR_KEY`
	if [ $? -ne 0 ]; then
		echo "Cannot load PVFS DS list from KV storage" 1>&2
		return 4
	fi

	echo "$ds_list" |dd of="$PVFS_DS_LIST_PATH" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Cannot write to $PVFS_DS_LIST_PATH" 1>&2
		return 5
	fi

	return 0
}

mpfs_fsal_configure()
{
	[ -n "$ENABLE_MPFS" ] || return 0

	mkdir -p "$MPFS_VAR_RUN"
	if [ $? -ne 0 ] ; then
		echo "Cannot create $MPFS_VAR_RUN" 1>&2
		return 1
	fi

	# Write mpfs_fsal.config
	local mpfs_fsal_config
	mpfs_fsal_config=$(pcs_generate_mpfs_fsal_config)
	if [ $? -ne 0 ]; then
		echo "Cannot generate MPFS FSAL config" 1>&2
		return 2
	fi

	echo "$mpfs_fsal_config" |dd of="$MPFS_FSAL_CONFIG_PATH" conv=fsync \
		>/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Cannot write to $MPFS_FSAL_CONFIG_PATH" 1>&2
		return 3
	fi

	return 0
}

pvfs_fsal_cleanup()
{
	[ -n "$ENABLE_PVFS" ] || return 0

	rm -f "$NFS_VAR_RUN/scripts"
	rm -f "$PVFS_DS_LIST_PATH"

	# If the service is stopped, there is nothing that could
	# unmount the vdisks, so we do it here.
	pcs_vdisk_unmount_all "$PVFS_VDISKS_MOUNT_DIR"
	if [ $? -ne 0 ] ; then
		echo "Unable to unmount ploop device(s)" 1>&2
		return 1
	fi

	rmdir "$PVFS_VDISKS_MOUNT_DIR"

	# Unregister after stopping the service and unmounting all vDisks,
	# since unregistering triggers vDisk remounting on other nodes.
	pcs_unregister_this_host_as_ds "$PVFS_DS_REGISTRY_KVSTOR_KEY"

	return 0
}

mpfs_fsal_cleanup()
{
	[ -n "$ENABLE_MPFS" ] || return 0

	rm -f "$MPFS_FSAL_CONFIG_PATH"
	return 0
}

pcs_ganesha_is_exported() {
	local export_id="$1"

	display_export=$($DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.DisplayExport \
	uint16:${export_id}  2>&1 | grep 'uint16 $export_id')

	if [ -z $display_export ]; then
		return 0
	fi

	echo " Ganesha export with id $export_id exists" 1>&2
	return 1
}

pcs_ganesha_gen_export_id() {
	while true; do
		#gen export_id in range from 1001-65535
		export_id=$(/usr/bin/shuf -i 1001-65535 -n 1)
		echo "export_id=$export_id"
		#check if current export_id already exists
		pcs_ganesha_is_exported $export_id
		[ $? -eq 0 ] && break;
	done
}

pcs_ganesha_add_export() {
	local export_id="$1"
	local config_name="$2"
	local config="$GANESHA_EXPORTS_ETC/$config_name"

	if [ ! -f $config ]; then
		echo "Cannot find export config '$config'" 1>&2
		return 1
	fi

	#add export to ganesha
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \
		string:$config string:"export(export_id=${export_id})" >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Cannot add export to ganesha" 1>&2
		return 2
	fi

	return 0
}

pcs_ganesha_del_export() {
	local export_id="$1"

	#delete export from ganesha
	$DBUS_SEND --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.RemoveExport uint16:$export_id >/dev/null 2>&1
}

pcs_nfs_down_export() {
	local share="$1"
	local export_name="$2"
	local export_path="$NFS_ROOT/shares/$share/exports/$export_name"
	local export_config="${share}_${export_name}.conf"
	local export_id=

	if [ -f "$export_path/export_id" ]; then
		export_id=`cat $export_path/export_id`
		if [ -z "$export_id" ] ; then
			echo "Unable to read export id for export '$export_path'" 1>&2
			return 1
		fi
	else
		echo "Unable find ganesha export id in export '$export_path'" 1>&2
		return 2
	fi

	pcs_ganesha_del_export $export_id

	rm -f $GANESHA_EXPORTS_ETC/$export_config >/dev/null 2>&1

	return $?
}

pcs_nfs_up_export() {
	local share="$1"
	local export_name="$2"
	local export_path="$NFS_ROOT/shares/$share/exports/$export_name"
	local export_config="${share}_${export_name}.conf"
	local export_id=

	# read export_id
	if [ -f "$export_path/export_id" ]; then
		export_id=`cat $export_path/export_id`
		if [ -z "$export_id" ] ; then
			echo "Unable to read export id for export '$export_path'" 1>&2
			return 1
		fi
	else
		echo "Unable find ganesha export id in export '$export_path'" 1>&2
		return 2
	fi

	# create local directory with configs
	mkdir -p $GANESHA_EXPORTS_ETC >/dev/null
	if [ $? -ne 0 ]; then
		echo "Cannot create $GANESHA_EXPORTS_ETC" 1>&2
		return 3
	fi

	# copy config from shared storage to local
	cp $export_path/$export_config $GANESHA_EXPORTS_ETC/
	if [ $? -ne 0 ]; then
		echo "Unable to copy config '$export_path/$export_config' to '$GANESHA_EXPORTS_ETC/'" 1>&2
		return 4
	fi

	pcs_ganesha_add_export $export_id $export_config
	if [ $? -ne 0 ]; then
		echo "Unable to add '$export_config' export to ganesha" 1>&2
		return 5
	fi

	return 0
}

pcs_nfs_parse_config()
{
	local tmp="$1"
	local share="$2"
	local export_config="$3"

	if [ ! -f "$export_config" ]; then
		echo "Unable find export config '$export_config'" 1>&2
		return 1
	fi

	# Config should have one export
	num_exports=`cat $export_config | grep -ioP "(?<=export_id).*" | wc -l`
	if [ $num_exports -ne 1 ]; then
		echo "Invalid number of exports in config '$export_config '. Export should be exaclty one per config" 1>&2
		return 2
	fi

	# Extract export id from config
	export_id=`cat $export_config | grep -ioP "(?<=export_id).*" | grep -ioP "\d+"`
	if [ -z "$export_id" ]; then
		echo "Can't extract export id from '$export_config'" 1>&2
		return 3
	fi

	# Save export_id
	pcs_ganesha_is_exported $export_id
	if [ $? -ne 0 ] ; then
		echo "Export with id $export_id already exported" 1>&2
		return 4
	fi
	echo $export_id | dd of="$tmp/export_id" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store file $tmp/export_id" 1>&2
		return 5
	fi

	# Extract and check share name
	local config_share=`cat $export_config | grep -ioP "(?<=share = ).*" \
			| grep -ioP "\w*"`
	if [ -z "$config_share" ]; then
		echo "Can't find share name in $export_config" 1>&2
		return 6
	fi
	if [ "$share" != "$config_share" ]; then
		echo "Share name in export config must be \"$share\"" \
				"instead of \"$config_share\"" 1>&2
		return 7
	fi

	return 0
}

# make export on shared storage
pcs_nfs_make_export() {
	local tmp="$1"
	local share="$2"
	local export_name="$3"
	local export_config="$4"

	if [ -z "$NFS_ROOT" ] ; then
		echo "NFS_ROOT must be specified" 1>&2
		return 1
	fi

	if [ ! -d "$NFS_ROOT/shares/$share/exports" ] ; then
		echo "Unable find directory $NFS_ROOT/shares/$share/shares" 1>&2
		return 2
	fi

	if [ -d "$NFS_ROOT/shares/$share/exports/$export_name" ]; then
		echo "Export '$export_name' already exist for share '$share'" 1>&2
		return 3
	fi

	# Set storage attributes
	stor_attr="mds-storage=1"
	pcs_nfs_set_storage_attr "$tmp" "$stor_attr"
	if [ $? -ne 0 ]; then
		echo "Can't set storage attributes $tmp $stor_attr" 1>&2
		rm -rf $tmp
		return 4
	fi

	pcs_nfs_parse_config $tmp $share $export_config
	if [ $? -ne 0 ] ; then
		echo "Can't parse config $export_config"
		return 5
	fi

	#cp ganeshas config to export directory
	cp "$export_config" "$tmp/${share}_${export_name}.conf"
	if [ $? -ne 0 ] ; then
		echo "Can't copy config $export_config to $tmp/${share}_${export_name}.conf"
		return 6
	fi

	mv "$tmp" "$NFS_ROOT/shares/$share/exports/$export_name"
	if [ $? -ne 0 ] ; then
		echo "Can't rename $tmp to $NFS_ROOT/shares/$share/exports/$export_name" 1>&2
		rm -rf $tmp >/dev/null 2>&1
		return 7
	fi

	return 0
}

pcs_nfs_make_share() {
	local name="$1"; shift
	local size="$1"; shift
	local addr="$1"; shift
	local vdisk_count="$1"; shift
	local vstorage_attrs=("$@")

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

	if [ -d "$NFS_ROOT/shares/$name" ] ; then
		echo "share '$name' already exist at $NFS_ROOT/shares" 1>&2
		return 2
	fi

	tmp=`mktemp -d "$NFS_ROOT/tmp/share-$name.XXXXXXXX"`
	if [ $? -ne 0 ] ; then
		echo "Can't create temporary directory." 1>&2
		return 4
	fi

	# Create share directories and set storage attributes
	for subdir in "control" "exports"; do
		pcs_nfs_create_dir_with_vstorage_attrs "$tmp/$subdir" "mds-storage=1"
		if [ $? -ne 0 ] ; then
			rm -rf $tmp
			return 10
		fi
	done

	pcs_nfs_create_dir_with_vstorage_attrs "$tmp/vdisks" "${vstorage_attrs[@]}"
	if [ $? -ne 0 ] ; then
		rm -rf $tmp
		return 11
	fi

	# Save version
	echo "$NFS_LIB_VERSION" | dd of="$tmp/control/version" conv=fsync >/dev/null 2>&1

	# Save address
	echo "$addr" | dd of="$tmp/control/address" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store address in file $tmp/control/address" 1>&2
		rm -rf $tmp
		return 12
	fi

	# Save mds_vdisk_id
	local mds_vdisk_id
	mds_vdisk_id=$(pcs_pvfs_vdisk_generate_id)
	if [ $? -ne 0 ] ; then
		echo "Can't generate a vDisk ID" >&2
		rm -rf $tmp
		return 13
	fi
	echo "$mds_vdisk_id" | dd of="$tmp/control/mds_vdisk_id" conv=fsync \
			>/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't write to $tmp/control/mds_vdisk_id" >&2
		rm -rf $tmp
		return 14
	fi

	# Save fs_id
	# NOTE: Using mds_vdisk_id as filesystem ID here, as it is:
	# - stable: doesn't change when a share is renamed;
	# - unique: changes when a share is deleted and created again
	echo "$mds_vdisk_id" | dd of="$tmp/control/fs_id" conv=fsync \
			>/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't write to $tmp/control/fs_id" >&2
		rm -rf $tmp
		return 14
	fi

	# Create MDS vdisk image
	pcs_pvfs_vdisk_create "$tmp" $mds_vdisk_id $size
	if [ $? -ne 0 ] ; then
		echo "Cannot create vDisk image" >&2
		rm -rf $tmp
		return 15
	fi

	# Create vdisks
	if [ -z "$vdisk_count" ]; then
		local ds_count=0
		while read line || [ -n "$line" ]; do
			ds_count=$(( $ds_count + 1 ))
		done < "$NFS_ROOT/ds.list"
		if [ "$ds_count" -eq 0 ] ; then
			echo "$NFS_ROOT/ds.list is empty?" >&2
			rm -rf $tmp
			return 31
		fi
		vdisk_count=$ds_count
	fi

	local ds_vdisk_size=$(( $size / $vdisk_count ))
	local vdisk_id
	local i
	for ((i = 1; i <= $vdisk_count; i++)); do
		# Create one vdisk per data server for now
		vdisk_id=$(pcs_pvfs_vdisk_generate_id)
		if [ $? -ne 0 ] ; then
			echo "Cannot generate a vDisk ID" >&2
			rm -rf $tmp
			return 34
		fi

		pcs_pvfs_vdisk_create "$tmp" $vdisk_id $ds_vdisk_size
		if [ $? -ne 0 ] ; then
			echo "Cannot create vDisk" >&2
			rm -rf $tmp
			return 35
		fi
	done

	mv $tmp "$NFS_ROOT/shares/$name"
	rc=$?
	if [ $? -ne 0 ]; then
		echo "Can't move $tmp to $NFS_ROOT/shares/$name" 1>&2
		rm -rf $tmp
		return 16
	fi

	pcs_nfs_register "$name"
	if [ $? -ne 0 ] ; then
		echo "Unable to register new share on current host" 1>&2
		return 17
	fi

	return 0
}

pcs_nfs_delete_share() {
	local name="$1"
	local force="$2"

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	[ $rc -gt 1 ] && return $rc
	if [ $rc -eq 0 ]; then
		# target registered on current host
		if [ -f "$NFS_SHM/.running/$name" ] ; then
			echo "Can't delete running share" 1>&2
			return 2
		fi
		pcs_nfs_unregister "$name"
	elif [ -z "$force" ]; then
		pcs_nfs_has_owner $name
		rc=$?
		[ $rc -gt 1 ] && return $rc

		if [ $rc -eq 1 ] ; then
			echo "share '$name' registered on another host" 1>&2
			return 3
		fi
	fi

	mv "$NFS_ROOT/shares/$name" "$NFS_ROOT/tmp/$name"
	if [ $? -ne 0 ] ; then
		echo "Can't move directory $NFS_ROOT/shares/$name to $NFS_ROOT/tmp/$name"
		return 4
	fi

	rm -rf $NFS_ROOT/tmp/$name

	return 0
}

mpfs_remove_share_from_mds() {
	local name="$1"
	local rc=0
	local path=""

	pcs_unregister_service "mpfs/mds" "mds-$name"
	if [ $? -ne 0 ]; then
		echo "Cannot unregister MDS for share '$name'" \
			"in Service Discovery" 1>&2
		rc=1
	fi

	path="$NFS_ROOT/shares/$name/control/mds_vdisk_id"
	local mds_vdisk_id=$(<"$path")
	if [ -z "$mds_vdisk_id" ] ; then
		echo "Unable to read $path" 1>&2
		return 1
	fi

	local errors
	errors=$($DBUS_SEND --print-reply --system \
		--dest=org.virtuozzo.VStorage.MPFS.MDS \
		/org/virtuozzo/VStorage/MPFS/MDS/Admin \
		org.virtuozzo.VStorage.MPFS.MDS.Admin.RemoveVolume \
		string:"$name" \
		2>&1 1>/dev/null)
	if [ $? -ne 0 ]; then
		case "$errors" in
		*org.virtuozzo.VStorage.MPFS.MDS.Error.NotFound*)
			echo "Volume '$name' is not found by MDS," \
				"already removed?" 1>&2
			;;
		*org.freedesktop.DBus.Error.ServiceUnknown*)
			echo "MDS is not available on DBus, not running?" 1>&2
			;;
		*)
			echo "$errors" 1>&2
			echo "Cannot remove volume '$name' from MDS" 1>&2
			# Fatal, because if the MDS cannot remove the volume,
			# we are not ready to unmout the vDisk
			return 1
			;;
		esac
	fi

	rm -f "$MPFS_MDS_VAR_RUN/$name.ds_vdisks"
	rm -f "$MPFS_MDS_VAR_RUN/$name.config"

	pcs_vdisk_unmount "$mds_vdisk_id" "$MPFS_MDS_VDISKS_MOUNT_DIR"
	if [ $? -ne 0 ] ; then
		echo "Unable to unmount metadata vDisk $mds_vdisk_id" \
				"from share '$name'" 1>&2
		rc=1
	fi

	return $rc
}

mpfs_add_share_to_mds() {
	local name="$1"

	local path="$NFS_ROOT/shares/$name/control/fs_id"
	local fs_id=$(<"$path")
	if [ -z "$fs_id" ] ; then
		echo "Unable to read $path" 1>&2
		return 1
	fi

	path="$NFS_ROOT/shares/$name/control/mds_vdisk_id"
	local mds_vdisk_id=$(<"$path")
	if [ -z "$mds_vdisk_id" ] ; then
		echo "Unable to read $path" 1>&2
		return 2
	fi

	local listen_address=$(pcs_get_config_value \
		"listen_address" "$MPFS_MDS_CONFIG")
	if [ -z "$listen_address" ]; then
		echo "Cannot get listen_address from $MPFS_MDS_CONFIG" 1>&2
		return 3
	fi

	local mds_rpc_port=$(echo $listen_address | cut -d: -f2)
	if [ -z "$mds_rpc_port" ]; then
		echo "Cannot parse listen_address '$listen_address'" \
			" from $MPFS_MDS_CONFIG" 1>&2
		return 4
	fi

	# Reset the registration first in case something fails
	pcs_unregister_service "mpfs/mds" "mds-$name"
	if [ $? -ne 0 ]; then
		echo "Cannot unregister MDS for share '$name'" \
			"in Service Discovery" 1>&2
		return 5
	fi

	pcs_vdisk_mount "$name" "$mds_vdisk_id" "$MPFS_MDS_VDISKS_MOUNT_DIR"
	if [ $? -ne 0 ] ; then
		echo "Unable to mount metadata vDisk $mds_vdisk_id" \
			"from share '$name'" 1>&2
		return 6
	fi

	mkdir -p "$MPFS_MDS_VAR_RUN"
	if [ $? -ne 0 ]; then
		echo "Cannot create $MPFS_MDS_VAR_RUN" 1>&2
		mpfs_remove_share_from_mds "$name"
		return 7
	fi

	local ds_vdisks_path="$MPFS_MDS_VAR_RUN/$name.ds_vdisks"
	pcs_list_ds_vdisks "$name" >"$ds_vdisks_path"
	if [ $? -ne 0 ]; then
		echo "Cannot write data vDisk list to '$ds_vdisks_path'" 1>&2
		mpfs_remove_share_from_mds "$name"
		return 8
	fi

	local volume_config_path="$MPFS_MDS_VAR_RUN/$name.config"
	{
		echo "fs_id = 0x$fs_id"
		echo "mds_vdisk_id = 0x$mds_vdisk_id"
		echo "ds_vdisk_list_path = $ds_vdisks_path"
	} >"$volume_config_path"
	if [ $? -ne 0 ]; then
		echo "Cannot write volume config to $volume_config_path" 1>&2
		mpfs_remove_share_from_mds "$name"
		return 9
	fi

	local errors
	errors=$($DBUS_SEND --print-reply --system \
		--dest=org.virtuozzo.VStorage.MPFS.MDS \
		/org/virtuozzo/VStorage/MPFS/MDS/Admin \
		org.virtuozzo.VStorage.MPFS.MDS.Admin.AddVolume \
		string:"$name" \
		string:"$volume_config_path" \
		2>&1 1>/dev/null)
	if [ $? -ne 0 ]; then
		case "$errors" in
		*org.virtuozzo.VStorage.MPFS.MDS.Error.AlreadyExists*)
			echo "Volume '$name' already exists in MDS" 1>&2
			errors=""
			;;
		*org.freedesktop.DBus.Error.ServiceUnknown*)
			echo "MDS is not available on DBus, not running?" 1>&2
			;;
		*)
			echo "$errors" 1>&2
			;;
		esac

		if [ -n "$errors" ]; then
			echo "Cannot add volume '$name' to MDS" 1>&2
			mpfs_remove_share_from_mds "$name"
			return 10
		fi
	fi

	pcs_register_service "mpfs/mds" --name "mds" --id "mds-$name" \
		--port "$mds_rpc_port" \
		--tag "fsname:$name" \
		--tag "fsid:$fs_id"
	if [ $? -ne 0 ]; then
		echo "Cannot register MDS for share '$name'" \
			"in Service Discovery" 1>&2
		mpfs_remove_share_from_mds "$name"
		return 11
	fi

	return 0
}

pcs_nfs_stop_share() {
	local name="$1"
	local force="$2"
	local scope="$3"
	local addr_ha=""
	local rc=0

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "The NFS share '$name' doesn't exist in $NFS_ROOT" 1>&2
                return 2
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "share '$name' is not registered on current host" 1>&2
		return $rc
	fi

	if [ -z "$force" ]; then
		if [ ! -f "$NFS_SHM/.running/$name" ]; then
			echo "The NFS share '$name' not running" 1>&2
			return 0
		fi
	fi

        if [ -f "$NFS_ROOT/shares/$name/control/address" ] ; then
                addr_ha=`cat "$NFS_ROOT/shares/$name/control/address"`
		pcs_ha_remove_addr $addr_ha
        fi

	#unmerge share keytab from global
	pcs_nfs_unmerge_keytab $name

	pcs_nfs_delete_share_cfg $name

	rm -f "$NFS_ROOT/shares/$name/control/.running"

        exports=`ls "$NFS_ROOT/shares/$name/exports" 2>/dev/null`
        for s in $exports; do
                [ ! -d "$NFS_ROOT/shares/$name/exports/$s" ] && continue
                pcs_nfs_down_export "$name" "$s"
        done

	rm -f "$NFS_SHM/.running/$name"

	local rc2
	if [ -z "$scope" ]; then
		mpfs_remove_share_from_mds "$name"
		rc2=$?
		if [ $rc2 -ne 0 ]; then
			rc=$(( 100 + $rc2 ))
		fi
	fi

	if [ -n "$ENABLE_PVFS" ]; then
		local vdisk_id=$(<"$NFS_ROOT/shares/$name/control/mds_vdisk_id")
		if [ -z "$vdisk_id" ] ; then
			echo "Unable to read mds_vdisk_id" \
				"from share '$name'" 1>&2
			rc=3
		fi

		if [ -n "$vdisk_id" ] ; then
			pcs_vdisk_unmount "$vdisk_id" "$PVFS_VDISKS_MOUNT_DIR"
			if [ $? -ne 0 ] ; then
				echo "Unable to unmount metadata vDisk" \
					"$vdisk_id from share '$name'" 1>&2
				rc=4
			fi
		fi
	fi

	return "$rc"
}

pcs_nfs_get_status() {
	local name="$1"
	if [ -z "$NFS_ROOT" -o ! -d  "$NFS_ROOT/shares/$name" ] ; then
		echo "unknown"
		return 1
	fi

	if [ ! -f "$NFS_SHM/.running/$name" ]; then
		echo "stopped"
		return 0
	fi

	echo "running"
	return 0
}

pcs_nfs_start_share() {
	local name="$1"
	local force="$2"  # force start even if already running
	local scope="$3"  # what to configure, empty means everything
	local addr=""
	local addr_ha=""
	local rc=0

        if [ -z "$NFS_ROOT" ] ; then
                echo "NFS_ROOT must be specified" 1>&2
                return 1
        fi

        if [ ! -d  "$NFS_ROOT/shares/$name" ] ; then
                echo "The NFS share '$name' doesn't exist in $NFS_ROOT" 1>&2
                return 2
        fi

	pcs_ha_is_registered_local "$name"
	rc=$?
	if [ $rc -ne 0 ] ; then
		[ $rc -eq 1 ] && echo "Share '$name' is not registered on current host" 1>&2
		return $rc
	fi

	[ ! -d "$NFS_SHM/.running" ] && mkdir -p "$NFS_SHM/.running"
	if [ -z "$force" ]; then
        	if [ -f "$NFS_SHM/.running/$name" ]; then
                	echo "Share '$name' already running." 1>&2
                	return 3
	        fi
	fi

	if [ -z "$scope" ]; then
		mpfs_add_share_to_mds "$name"
		rc=$?
		if [ $rc -ne 0 ]; then
			return $(( 100 + $rc ))
		fi
	fi

	addr_ha=`cat "$NFS_ROOT/shares/$name/control/address"`
        if [ -z "$addr_ha" ] ; then
                echo "Unable to read address for share '$name'" 1>&2
                return 4
        fi

	pcs_nfs_write_share_cfg $name
	if [ $? -ne 0 ] ; then
                echo "Unable to write config for share '$name'" 1>&2
                return 5
        fi

	if [ -n "$ENABLE_PVFS" ]; then
		local vdisk_id=$(<"$NFS_ROOT/shares/$name/control/mds_vdisk_id")
		if [ -z "$vdisk_id" ] ; then
			echo "Unable to read mds_vdisk_id" \
				"from share '$name'" 1>&2
			return 21
		fi

		pcs_vdisk_mount "$name" "$vdisk_id" "$PVFS_VDISKS_MOUNT_DIR"
		if [ $? -ne 0 ] ; then
			echo "Unable to mount vDisk $vdisk_id" \
				"from share '$name'" 1>&2
			return 22
		fi
	fi

	exports=`ls "$NFS_ROOT/shares/$name/exports" 2>/dev/null`
	for s in $exports; do
		[ ! -d "$NFS_ROOT/shares/$name/exports/$s" ] && continue
		pcs_nfs_up_export "$name" "$s"
		if [ $? -ne 0 ] ; then
			pcs_nfs_stop_share "$name" 1 "$scope"
			return 6
		fi
	done

	#merge share keytab into global
	pcs_nfs_merge_keytab $name
	if [ $? -ne 0 ] ; then
		pcs_nfs_stop_share "$name" 1 "$scope"
		return 7
	fi

	# enable grace period before adding IP
	addr=`echo $addr_ha | cut -d '/' -f1`
	pcs_nfs_start_grace "$addr"
	if [ $? -ne 0 ] ; then
		pcs_nfs_stop_share "$name" 1 "$scope"
		return 8
	fi

	pcs_ha_add_addr $addr_ha
	if [ $? -ne 0 ] ; then
		pcs_nfs_stop_share "$name" 1 "$scope"
		return 9
	fi

	echo "1" | dd of="$NFS_ROOT/shares/$name/control/.running" conv=fsync >/dev/null 2>&1
	if [ $? -ne 0 ] ; then
		echo "Can't store file $NFS_ROOT/shares/$name/control/.running" 1>&2
		pcs_nfs_stop_share "$name" 1 "$scope"
		return 10
	fi

        echo "1" > "$NFS_SHM/.running/$name"

	return 0
}

for_each_local_share() {
	local action=$1
	local retval=0

	local host_id=`get_host_id`
	if [ -z "$host_id" ] ; then
		echo "Can't get host id" 1>&2
		return 1
	fi

	ls $NFS_ROOT/shares 2>/dev/null | while read share
	do
		[ ! -d "$NFS_ROOT/shares/$share/control" ] && continue

		pcs_ha_is_registered_local $share $host_id >/dev/null 2>&1
		local rc=$?
		if [ $rc -eq 0 ] ; then
			load_matching_nfs_lib_version "$share"
			pcs_ha_lock_exec $action $share
			[ $? -ne 0 ] && retval=1
		else
			[ $rc -ne 1 ] && retval=$rc
		fi
	done

	return $retval
}
