#!/usr/bin/bash
# Copyright (c) 1999-2017, Parallels International GmbH
# Copyright (c) 2017-2019 Virtuozzo International GmbH. All rights reserved.
#
# This file is part of OpenVZ libraries. OpenVZ is free software; you can
# redistribute it and/or modify it under the terms of the GNU Lesser General
# Public License as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
#
# Our contact details: Virtuozzo International GmbH, Vordergasse 59, 8200
# Schaffhausen, Switzerland.
#
# Common stuff for vzctl helper scripts
# get the name of the script
SELFNAME=`basename $0`
# Set the sane umask
umask 022
export PATH=/sbin:/bin:/usr/sbin:/usr/bin

# Error codes
VZ_INVALID_PARAMETER_SYNTAX=20
VZ_FS_NO_DISK_SPACE=46
VZ_FS_BAD_TMPL=47
VZ_FS_NEW_VE_PRVT=48
VZ_CHANGEPASS=74
VZ_CANT_ADDIP=34
VZ_IP_INUSE=78
VZ_SET_RATE=80
VZ_SET_ACCOUNT=81

# iptables parameters
VE_STATE_DIR="/var/vz/veip/"
CONF_DIR=/etc/vz/conf
VZ_IP_CLASSES="${CONF_DIR}/networks_classes"
PS_START_LOCK_FILE="/var/lock/subsys/ps_start_lock"
PRAM_MOUNT_POINT="/mnt/pram"

# tc parameters
TC_CMD=tc
IP_CMD=ip
NFT_CMD=nft
MAX_FORKS=100
MAX_CLASSES=16
ARPING_CMD="/sbin/arping"
ARPING_ARGS="-f -c 1 -q"
NDSEND_CMD="/usr/sbin/ndsend"
OVSCTL_CMD="/usr/bin/ovs-vsctl"
VZ_TC_CLASSES="/var/run/vz_tc_classes"
VZ_TC_CLASSES_LOCK="/var/lock/vz_tc_classes.lock"
VZ_ARP_ANNOUNCE_LOCK="/var/lock/vz_arp_announce.lock"

# interfaces parameters
ALLOT="allot 3028"
CELL="cell 8"
MPU="mpu 64"
PRIO="prio 1"
PRIOV6="prio 2"

let AVPKT=1000
let HZ=0

# inter functions variables
vzget_ve_parameter_RET=""
vzget_tc_class_RET=""
# Prints error message and exits
# Parameters:
#   $1 - error message
#   $2 - exit code
# Example of usage:
#   vzerror "Fatal error" 1
function vzerror()
{
	ERR=$?
	echo "$SELFNAME ERROR: $1" >&2
	exit $2
}

# Prints warning message
# Parameters:
#   $* - error message
# Example of usage:
#   vzwarning Invalid user
function vzwarning()
{
	echo "$SELFNAME WARNING: $*"
}

# Prints debug message
# Parameters:
#   $* - debug message
# Example of usage:
#   vzdebug Trying to start ls
function vzdebug()
{
	echo "$SELFNAME: $*"
}

# Executes a command and logs this fact, with command exit code.
# Returns exit code of executed command.
# dx stands for DebugExec
# Parameters:
#   $* - command to execute
# Example of usage:
#   vzdx ls -l
function vzdx()
{
	vzdebug "EXEC $*"
	eval $*
	EXIT_CODE=$?
	vzdebug "EXIT_CODE $EXIT_CODE"
	return $EXIT_CODE
}

# Checks if environment variable exists,
# and exits with exit code 1 if not
# Parameters:
#   $* - option names
# Example:
#   vzcheckvar VEID IP_ADDR
function vzcheckvar()
{
	for VAR in $*; do
		if eval test "\"x\$$VAR\"" = "x"; then
			vzerror "Missing parameter: $VAR" $VZ_INVALID_PARAMETER_SYNTAX
		fi
	done
}

# Open/close the host-side veth of a routed CT in the host firewall, so the
# host (acting as router for the CT) accepts its traffic. Two backends are
# supported, selected by VE_FIREWALL_BACKEND in /etc/vz/vz.conf:
#
#   iptables  - openings live as ACCEPT rules in the agent-owned VZ_*_VE chains
#               (VHI: the host firewall is iptables-nft managed by
#               vstorage-ui-agent; firewalld is masked).
#   firewalld - put the veth into firewalld's 'trusted' zone.
#   none      - do nothing (firewalling handled externally).
#   auto      - (default) use firewalld if it is running, otherwise iptables.
#
# vzfw_backend() resolves 'auto'/empty to a concrete backend per call.
VE_IPT_WAIT="-w 30"
VE_IN_CHAIN="VZ_IN_VE"
VE_OUT_CHAIN="VZ_OUT_VE"
VE_FWD_CHAIN="VZ_FWD_VE"

# Resolve and echo the firewall backend. On the first call, if VE_FIREWALL_BACKEND
# is 'auto' or unset, probe firewalld once and cache the concrete backend back
# into VE_FIREWALL_BACKEND so the probe is not repeated.
vzfw_backend()
{
	if [ "${VE_FIREWALL_BACKEND:-auto}" = "auto" ]; then
		if firewall-cmd --state >/dev/null 2>&1; then
			VE_FIREWALL_BACKEND=firewalld
		else
			VE_FIREWALL_BACKEND=iptables
		fi
	fi
	echo "$VE_FIREWALL_BACKEND"
}

# --- iptables backend (agent-owned VZ_*_VE chains) ---------------------------
# The agent (python3-vstorage) owns the VZ_IN_VE / VZ_OUT_VE / VZ_FWD_VE chains,
# links them before its REJECT tails, and leaves their contents to us. We must
# not assume they already exist (cold boot, old agent, flushed table), so
# vzct_ensure_ve_chains creates+links them first. All ops are idempotent and
# take the xtables lock so we don't collide with the agent.
vzct_ensure_ve_chains()
{
	local ipt base_chain base chain
	for ipt in iptables ip6tables; do
		for base_chain in "INPUT:$VE_IN_CHAIN" "OUTPUT:$VE_OUT_CHAIN" "FORWARD:$VE_FWD_CHAIN"; do
			base=${base_chain%%:*}
			chain=${base_chain##*:}
			$ipt $VE_IPT_WAIT -t filter -nL "$chain" >/dev/null 2>&1 || \
				$ipt $VE_IPT_WAIT -t filter -N "$chain" || \
				vzerror "Unable to create $ipt chain $chain" $VZ_CANT_ADDIP
			$ipt $VE_IPT_WAIT -t filter -C "$base" -j "$chain" 2>/dev/null || \
				$ipt $VE_IPT_WAIT -t filter -I "$base" 1 -j "$chain" || \
				vzerror "Unable to link $ipt chain $chain into $base" $VZ_CANT_ADDIP
		done
	done
}

# The veth openings, one per "chain:direction": accept the CT veth as input in
# VZ_IN_VE, as output in VZ_OUT_VE, and both directions in VZ_FWD_VE.
VE_RULES="$VE_IN_CHAIN:-i $VE_OUT_CHAIN:-o $VE_FWD_CHAIN:-i $VE_FWD_CHAIN:-o"

vzaddtrustedfw_iptables()
{
	local dev=$1 ipt rule chain dir
	vzct_ensure_ve_chains
	for ipt in iptables ip6tables; do
		for rule in $VE_RULES; do
			chain=${rule%%:*}
			dir=${rule##*:}
			$ipt $VE_IPT_WAIT -t filter -C "$chain" $dir "$dev" -j ACCEPT 2>/dev/null && continue
			$ipt $VE_IPT_WAIT -t filter -A "$chain" $dir "$dev" -j ACCEPT || \
				vzerror "Unable to add $ipt rule: $chain $dir $dev -j ACCEPT" $VZ_CANT_ADDIP
		done
	done
}

vzdeltrustedfw_iptables()
{
	local dev=$1 ipt rule chain dir
	for ipt in iptables ip6tables; do
		for rule in $VE_RULES; do
			chain=${rule%%:*}
			dir=${rule##*:}
			$ipt $VE_IPT_WAIT -t filter -D "$chain" $dir "$dev" -j ACCEPT 2>/dev/null
		done
	done
}

# --- firewalld backend 'trusted' zone -------------------------------
vzaddtrustedfw_firewalld()
{
	local i=1 res=1 answ=''

	while [ $i -le 5 ] && [ $res -ne 0 ];
	do
		answ=$( firewall-cmd --zone=trusted --add-interface=$1 2>&1)
		res=$?

		if [ $i -lt 5 ] && [ $res -ne 0 ]; then
			echo "firewall-cmd finished with code: $res, firewall-cmd has been run again"
			sleep 10
		fi

		((i++))
	done

	if [ $res -ne 0 ]; then
		vzerror "After $i tries, firewall-cmd has been returned non-zero code: $res, firewall-cmd finished 
		with message: $answ,  Unable to add host routed $1 into the trusted firewall zone in runtime " -1
	fi

	firewall-cmd --permanent --zone=trusted --add-interface=$1
	if [ $? -ne 0 ]; then
		vzerror "Unable to add host routed $1 into the trusted firewall zone permanent " -1
	fi
}

vzdeltrustedfw_firewalld()
{
	local v=$1 isfw
	isfw=`firewall-cmd --zone=trusted --query-interface=$v 2>/dev/null`
	if [ "$isfw" = "yes" ]; then
		firewall-cmd --zone=trusted --remove-interface=$v
		firewall-cmd --permanent --zone=trusted --remove-interface=$v
	fi
}

# --- dispatchers (the names the rest of libvzctl calls) ----------------------
# $1 = host-side interface name (HNAME).
vzaddtrustedfw()
{
	case "$(vzfw_backend)" in
		firewalld) vzaddtrustedfw_firewalld "$1" ;;
		none)      : ;;
		*)         vzaddtrustedfw_iptables  "$1" ;;
	esac
}

vzdeltrustedfw()
{
	case "$(vzfw_backend)" in
		firewalld) vzdeltrustedfw_firewalld "$1" ;;
		none)      : ;;
		*)         vzdeltrustedfw_iptables  "$1" ;;
	esac
}

# Creates a pair of veth devices, being one of them in the host, and the other
# inside the container The function should be called with the following env
# variables set:
#
#  HNAME: The intended name of the device in the host side
#  VNAME: The indented name of the device inside the container
#  VEID:  The numerical container id.
#
# HNAME and VNAME must be different, since they will at some point cohexist at
# the host side, and our name convention forced by vzctl tool guarantees that
# this shouldn't be a problem for sane setups.
#
# This function leaves the host device in the "link up" state, but with no
# further configuration. The container device setup is left entirely to the
# container.
#
# If the device given by HNAME in the host already exists, this function exits
# with no visible action.
vzcreatepair()
{
	${IP_CMD} link show type veth dev $HNAME >/dev/null 2>&1
	if [ $? -eq 0 ]; then
		${IP_CMD} -n $VEID link show type veth dev $VNAME >/dev/null 2>&1
		if [ $? -eq 0 ]; then
			${IP_CMD} link set $HNAME up
			return 0
		fi

		# precreate veth case
		v=`${IP_CMD} link show type veth dev $HNAME 2>/dev/null | awk '/'"$HNAME"'@/{split($2, a, "[@:]"); print a[2]}'`
		h=`${IP_CMD} link show type veth dev $v 2>/dev/null | awk '/'"$v"'@/{split($2, a, "[@:]"); print a[2]}'`
		if [ -n "$v" -a "$h" = "$HNAME" ]; then
			${IP_CMD} link set $v netns $VEID || \
				vzerror "Unable to move veth interface $v to Container ns" -1
			${IP_CMD} -n $VEID link set dev $v name $VNAME || \
				vzerror "Unable to rename $v to $VNAME" -1
			${IP_CMD} -n $VEID link set dev $VNAME up
			${IP_CMD} link set $HNAME up
			if [ "$NETWORK_TYPE" = "routed" ]; then
				sysctl -w net.ipv4.conf.${HNAME}.proxy_arp=1 ||\
					vzerror "Unable to turn on proxy_arp on $HNAME" 1
				${IP_CMD} a a fe80::ffff:1:1/128 dev ${HNAME}
				vzaddtrustedfw $HNAME
			fi
			return
		fi

		# Due to races, interfaces may exist some time after VE stop. Try to remove.
		echo Only host part $HNAME of veth pair exists
		${IP_CMD} link delete $HNAME || echo "Unable to delete veth interface $HNAME"
	fi

	if [ "$VNAME" = "venet0" ]; then
		if ${IP_CMD} -n $VEID l l dev venet0 2>/dev/null | grep -q POINTOPOINT; then
			${IP_CMD} -n $VEID l s dev venet0 name xvenet0
		fi
	fi
	${IP_CMD} -n $VEID link add name $HNAME type veth peer name $VNAME || \
		vzerror "Unable to create veth pair ($HNAME, $VNAME)" -1
	${IP_CMD} -n $VEID link set $HNAME netns $$ || \
		vzerror "Unable to move veth interface $HNAME to host ns" -1
	${IP_CMD} -n $VEID link set dev $VNAME up
	${IP_CMD} link set $HNAME up
	if [ "$NETWORK_TYPE" = "routed" ]; then
		sysctl -w net.ipv4.conf.${HNAME}.proxy_arp=1 ||\
			vzerror "Unable to turn on proxy_arp on $HNAME" 1
		${IP_CMD} a a fe80::ffff:1:1/128 dev ${HNAME}
		vzaddtrustedfw $HNAME
	fi
}

# Move an already existing interface to the network namespace given by VEID
vzmoveif()
{
	${IP_CMD} link show | grep -w -F $HNAME >/dev/null 2>&1
	if [ $? -ne 0 ]; then
		vzerror "Can't move non-existent interface"
		return;
	fi

	${IP_CMD} link set $HNAME netns $VEID
}

# Removes the interface referred to by VNAME from the container.
vzdestroylink()
{
	${IP_CMD} -n $VEID link delete $VNAME type veth
}

# desired mac addresses.  The host mac address is given by HMAC, and the
# container's by VMAC.  They are both optional parameters. If no mac is
# specified, this function does nothing.
vzadjustmacs()
{
	if [ "x$HMAC" != "x" ]; then
		${IP_CMD} link set dev $HNAME address $HMAC
	fi

	[ "x$VMAC" = "x" ] && return

	o=`${IP_CMD} -n $VEID link list dev $VNAME | \
		awk '/link\/ethe/{print toupper($2)}'`

	if [ "$VMAC" != "$o" ]; then
		${IP_CMD} -n $VEID link set dev $VNAME address $VMAC
	fi
}

# Bridge configuration function.
#
# It allows us to automatically insert a given host-side interface (HNAME) into
# a bridge. Calling the bridge "venet0" has a special meaning, since this is a
# bridge expected to always exist in our setup. In this case, we'll make sure
# the bridge exists by creating it in case we are the first caller.  For all
# other setups, the bridge is expected to already exist and be valid.
vzconfbridge()
{
	if [ "x$BRIDGE" = "xvenet0" ]; then
		if [ $(brctl show venet0 2>/dev/null | tail -n+2 | wc -l) -eq 0 ]; then
			brctl addbr venet0
			${IP_CMD} link set venet0 up
		fi
	fi

	if [ "x$BRIDGE" != "x" ]; then
		brctl addif $BRIDGE $HNAME >/dev/null 2>&1
	fi
}

# Move back an interface to the host namespace.
#
# When we move a physical host-side network device to the container, this is
# what we should call to undo the operation.
vzregainif()
{
	${IP_CMD} -n $VEID link set $VNAME netns $$
}

# This function fills $NETDEVICES with all network interfaces
# You should always call it before calling vzarp
function vzgetnetdev()
{
# Get a list of interfaces, excluding ones with LOOPBACK NOARP or SLAVE flags
# Accept mask: eth*[0-9]|tr*[0-9]|wlan[0-9]|ath[0-9]|ip6tnl*[0-9]|mip6mnha*[0-9]|bond[0-9]
	NETDEVICES=`${IP_CMD} a l | awk '
/^[[:digit:]]+:/ && /\<UP\>/ && !/\<LOOPBACK\>/ && !/\<SLAVE\>/ && !/\<NOARP\>/{
	print $2
}' | sed -e 's/:$//' -e 's/@.*//'`

	local tmp_list=""

	# filter out devices without IP address
	tmp_list=""
	for dev in $NETDEVICES; do
		if ! [[ -e "/sys/class/net/$dev/device" \
			|| -e "/sys/class/net/$dev/bridge" \
			|| -e "/proc/net/vlan/$dev" \
			|| $dev =~ ^bond ]]; then
			continue
		fi
		local tmp=`${IP_CMD} a l $dev | awk -v dev=$dev \
			'/^^[[:blank:]]+inet6? / && !/scope link/ { print(dev); exit; }'`
		[ -n "$tmp" ] && tmp_list="$tmp_list $tmp"
	done
	NETDEVICES="$tmp_list"
}

# Adds/deletes public ARP records for given IP for all interfaces
# Parameters:
#   $1		- should be either "add" or "del"
#   $2		- IP address
#   $NETDEVICES - Network devices used to take MAC addresses from
function vzarp()
{
	local DEV
	local devlist

	if [ "$1" = "del" ]; then
		if [ "${2#*:}" != "${2}" ]; then
			# use $NETDEVICES for ipv6 (#PCLIN-30830, #PCLIN-30753)
			devlist=${NETDEVICES}
		else
			devlist=`awk '/^'$2' / {print $6}' /proc/net/arp`
		fi
	else
		devlist=${NETDEVICES}
	fi

	for DEV in $devlist; do
		${IP_CMD} neigh $1 proxy $2 dev $DEV > /dev/null 2>&1
	done
}

clean_routed_ips()
{
      if [ -f "$VE_STATE_DIR/$VEID" ]; then
              [ -z "$NETDEVICES" ] && vzgetnetdev
              for ip in `cat $VE_STATE_DIR/$VEID 2>/dev/null`; do
                      ip=${ip%%/*}
                      vzdelrouting $ip
                      vzarp del $ip
              done
              > $VE_STATE_DIR/$VEID
      fi
}

# Runs a command for every element of an argument array, in batches of
# $MAX_FORKS parallel children.
# Parameters:
#   $1		- delay in seconds between the checks of a running child
#   $2		- command to run
#   $3		- name of the array with the per-child arguments
# Every child that exits non-zero is reported twice: as a warning, and as one
# "<exit code> <arguments>" entry in $PARALLEL_FAILED. A caller that can tell
# a real conflict from a tool failure reads $PARALLEL_FAILED, see
# vzarpipdetect(). The return value stays 0, because most callers send
# best-effort notifications and must not stop on a failed child.
function parallel_execution()
{
	local counter=0
	local fork_counter=0
	local delay="${1}"
	local ex="${2}"
	local pids=""
	local cmd=""
	local rc=0
	declare -a args=("${!3}")
	declare -A arg_list=()

	PARALLEL_FAILED=()

	while ((${counter} < ${#args[@]})); do
		fork_counter=${MAX_FORKS}
		pids=""
		while ((${fork_counter} > 0 && ${counter} < ${#args[@]})); do
			cmd="${ex} ${args[${counter}]}"
			${cmd} &
			command_list[$!]="${cmd}"
			arg_list[$!]="${args[${counter}]}"
			pids="${pids} $!"
			counter=$((${counter}+1))
			fork_counter=$((${fork_counter}-1))
		done

		while [ ! -z "${pids}" ]; do
			for pid in ${pids}; do
				kill -0 ${pid} > /dev/null 2>&1
				[ $? -eq 0 ] && sleep ${delay} && continue
				wait ${pid}
				rc=$?
				if [ ${rc} -ne 0 ]; then
					vzwarning "${command_list[${pid}]} FAILED"
					PARALLEL_FAILED+=("${rc} ${arg_list[${pid}]}")
				fi
				pids=$(echo ${pids} | sed "s,\<${pid}\>,,g")
			done
		done
	done
}

# Send ARP request to detect that somebody already have this IP.
# Exits with $VZ_IP_INUSE when a host on the network answers for an address.
function vzarpipdetect()
{
	local dev
	local ip
	local ips
	local entry
	local rc
	local ipv4_pairs=()

	ips=${1}

	if [ ! -x ${ARPING_CMD} ]; then
		echo "There is no ${ARPING_CMD}!"
		return 1
	fi
	[ "${SKIP_ARPDETECT}" = "yes" ] && return
	[ -z "${ips}" ] && return

	for dev in ${NETDEVICES}; do
		for ip in ${ips}; do
			if [ $(expr index ${ip} ":") -eq 0 ]; then
				ipv4_pairs+=("${dev} ${ip}")
			fi
		done
	done
	# Duplicate address detection mode (DAD). Returns 0, if DAD succeeded i.e. no replies are received
	parallel_execution 1 "${ARPING_CMD} ${ARPING_ARGS} -w 1 -D -I" ipv4_pairs[@]

	# arping exits 1 when it gets a reply, that is, when another host on the
	# network answers for the address. It exits 2 or more on its own errors,
	# for example on a missing device. Only a reply is a conflict; an arping
	# error must not stop the container.
	for entry in "${PARALLEL_FAILED[@]}"; do
		read -r rc dev ip <<< "${entry}"
		[ "${rc}" != "1" ] && continue
		vzerror "IP ${ip} already in use on the network (reply on ${dev})." \
			${VZ_IP_INUSE}
	done
}

# Send ARP request to update neighbour ARP caches
function vzarpipset()
{
	local dev
	local ip
	local ips=${1}
	local binary
	local ipv4_pairs=()
	local ipv6_pairs=()

	[ -z "${ips}" ] && return

	for binary in ${ARPING_CMD} ${NDSEND_CMD}; do
		if [ ! -x ${binary} ]; then
			echo "There is no ${binary}"
			return 1
		fi
	done

	for dev in ${NETDEVICES}; do
		for ip in ${ips}; do
			if [ $(expr index ${ip} ":") -eq 0 ]; then
				ipv4_pairs+=("${dev} ${ip}")
			else
				ipv6_pairs+=("${ip} ${dev}")
			fi
		done
	done

	# ip_nonlocal_bind option allows host to announce arbitrary IP address
	# so we enable it globally to prevent arping error
	# (bind: cannot assign requested address)
	lockfile ${VZ_ARP_ANNOUNCE_LOCK}
	nonlocal_bind_prev=$(cat /proc/sys/net/ipv4/ip_nonlocal_bind)
	echo "1" > /proc/sys/net/ipv4/ip_nonlocal_bind
	parallel_execution 1 "${ARPING_CMD} ${ARPING_ARGS} -A -I" ipv4_pairs[@]
	echo "${nonlocal_bind_prev}" > /proc/sys/net/ipv4/ip_nonlocal_bind
	unlockfile
	parallel_execution 1 "${NDSEND_CMD}" ipv6_pairs[@]
}

# Sets VE0 source routing for given IP
# Parameters:
#   $1 - IP address
function vzaddrouting()
{
	local dev=$1
	local ip=$2
	local src_addr=
	local cmd=

	if [ -n "${VE_ROUTE_SRC_DEV}" -a "${1#*:}" = "${1}" ]; then
		src_addr=`ip route list table local dev ${VE_ROUTE_SRC_DEV} | \
			grep '^local'| \
			grep -v "dev venet0" | cut -d' ' -f2 | \
			grep -v '^127\.' | head -1`
		if [ -z "${src_addr}" ]; then
			vzerror "Unable to get source ip [$VE_ROUTE_SRC_DEV]" $VZ_CANT_ADDIP
		fi
		src_addr="src ${src_addr}"
	fi
	cmd="${IP_CMD} route replace $ip dev $dev ${src_addr}"
	${cmd} || vzerror "Unable to set route: ${cmd}" $VZ_CANT_ADDIP
}

# Deletes VE0 source routing for given IP
# Parameters:
#   $1 - IP address
function vzdelrouting()
{
	local rc=0
	local fmt=
	local arg="route del"

	if [ "${1%%:*}" != "${1}" ]; then
		fmt=-6
		arg="-6 route flush"
	fi

	while [ $rc -eq 0 ]; do
		out=`${IP_CMD} $fmt route list "$1"`
		rc=$?
		if [ $rc -ne 0 ]; then
			vzwarning "Failed to get route list: ${IP_CMD} $fmt route list $1 [$rc]"
		elif ! echo $out | grep -qw "$1"; then
			break
		fi

		${IP_CMD} $arg "$1"
		if [ $? -ne 0 ]; then
			vzwarning "Unable to remove route: ${IP_CMD} $arg $1"
			break
		fi
	done
}

# Checks if all items from the first list
# are in the second one.
# Parameters:
#   $1 - the first list
#   $2 - the second list
function vzis_list_in_list()
{
	local found;
	local i;
	local j;

	[ -z "${1}" -a -z "${2}" ] && return 1;

	for i in ${1} ; do
		let found=0;
		for j in ${2} ; do
			if [ "X${i}" = "X${j}" ]; then
				let found=1;
				break;
			fi
		done
		if [ ${found} -eq 0 ]; then
			echo "${i}"
			return 0;
		fi
	done
	return 1;
}

# Makes list of the first fields (: separated)
# of $1
# Parameters:
#   $1 - source list
function vzmk_list1()
{
	local i;

	RET_LIST=

	[ -z "${1}" ] && return;

	for i in ${1} ; do
		RET_LIST="${RET_LIST} ${i%%:*}"
	done
}

# Makes list of the second fields (: separated)
# of $1
# Parameters:
#   $1 - source list
function vzmk_list2()
{
	local TMP;
	local i;

	RET_LIST=

	[ -z "${1}" ] && return;

	for i in ${1} ; do
		TMP=${i#*:}
		RET_LIST="${RET_LIST} ${TMP%%:*}"
	done
}

# Takes an exclusive flock on ${1} and keeps the descriptor in __LOCK_FD for
# unlockfile(). The lock lives on the open file, so the kernel drops it when
# the holder exits however it exits, and the file itself is never removed:
# unlinking it would let the next caller lock a fresh inode and walk straight
# into the section this one is in.
function lockfile()
{
	local LOCKFILE="${1}"

	[ -n "${__NOTLOCK}" ] && return 0
	# Braces scope the redirect to this exec. A bare exec keeps the redirect for the rest of the script.
	{ exec {__LOCK_FD}> ${LOCKFILE}; } 2> /dev/null || {
		vzerror "Can't open ${LOCKFILE}." ${VZ_SET_ACCOUNT}
	}
	flock ${__LOCK_FD} || {
		vzerror "Can't lock ${LOCKFILE}." ${VZ_SET_ACCOUNT}
	}
}

function unlockfile()
{
	[ -n "${__LOCK_FD}" ] && exec {__LOCK_FD}>&-
	unset __LOCK_FD
}

function umount_pram()
{
	umount "$PRAM_MOUNT_POINT"
}

#mount pram as file system
function mount_pram()
{
	local opts=$1

	# detect where to save CT dump
	if [ ! -f "/sys/kernel/pram" ] ; then
		echo "Warning: pram in not availabe"
		return 1
	fi

	if [ "$opts" = "init" ] ; then
		opts="-o noload"
	fi

	[ -d $PRAM_MOUNT_POINT ] || mkdir -p $PRAM_MOUNT_POINT

	# mount pram
	mount ${opts} -t pram none $PRAM_MOUNT_POINT
	if [ $? -ne  0 ] ; then
		echo "Warinig: Failed to mount pram"
		return 1
	fi

	return 0
}

# Compute and return TCID from current VEID
# Parameters:
# No
#
# ATTENTION! If you modify this function you MUST also
# modify the same code in file "env.c" in function "ctid2veid"!
function vzget_tcid()
{
	if [[ "$VEID" =~ .+-.+ ]]; then
		echo $((0x${VEID%%-*} & 0x7fffffff))
	else
		echo $((${VEID} & 0x7fffffff))
	fi
}

# Compute and return VEID without possible symbols "-".
# Parameters:
# No
#
# ATTENTION! If you modify this function you MUST also
# modify the same code in file "net.c" in function "ctid2nft"!
function vzget_nft()
{
	echo ${VEID//-/}
}

# Compute and return mark value from class
# Parameters:
# $1 - class
# $2 - tc class (HEX format)
function vzget_mark()
{
	echo $((0x${2}*2*${MAX_CLASSES}+${1}))
}

# Executes a command.
# Returns exit code of executed command or
# exits with ${VZ_SET_RATE}.
# Parameters:
#   $* - command to execute
vzrun()
{
	local EXIT_CODE

	($*)
	EXIT_CODE=$?
	if [ "X${EXIT_CODE}" != "X0" ]; then
		vzerror "Error while executing: <<$*>>." ${VZ_SET_RATE}
	fi
}

# Works under VZ_TC_CLASSES_LOCK.
# Reads TC_CLASS for TCID
# Parameters:
#   $1 - TCID
vzread_tc_class()
{
	local tmp;

	lockfile ${VZ_TC_CLASSES_LOCK}
	tmp=`grep "^${1} " ${VZ_TC_CLASSES} 2> /dev/null | cut -f2 -d' '`
	unlockfile

	vzread_tc_class_RET="${tmp}"
}

# Checks classes in list
# Parameters:
#   $1 - classes list
vzcheck_classes()
{
	local i;

	[ -z "${1}" ] && return 1;

	for i in ${1} ; do
		if [ ${i} -ge ${MAX_CLASSES} ]; then
			vzwarning "Wrong class (${1}). MAX_CLASSES=${MAX_CLASSES}."
			return 0;
		fi
		if [ ${i} -le 0 ]; then
			vzwarning "Wrong class (${1}). Must be > 0."
			return 0;
		fi
	done
	return 1;
}

# Reads IP classes list from VZ_IP_CLASSES
# Parameters:
# No
vzget_classes_list()
{
	if [ ! -f ${VZ_IP_CLASSES} ]; then
		if [ "X${TRAFFIC_SHAPING}" = "Xyes" ]; then
			vzerror "Can't find <<${VZ_IP_CLASSES}>>. Traffic accounting/shaping operations are impossible." ${VZ_SET_ACCOUNT}
		else
			CLASSES_LIST=""
			return
		fi
	fi

	CLASSES_LIST=`grep  '^[1-9]' ${VZ_IP_CLASSES} | cut -f 1 -d ' '| sort | uniq`
	vzcheck_classes "${CLASSES_LIST}"
	[ ! $? -eq 1 ] &&  vzerror "Wrong class value in ${VZ_IP_CLASSES}." ${VZ_SET_ACCOUNT}
}

# Create nft rules for marking network packets
# Parameters:
# $1 - class
# $2 - mark
# $3 - veth list
vzcreate_nft_rules()
{
	local class=$1
	local mark=$2
	local veth=$3
	local ctid=$(vzget_nft)
	local class_ips vdev dev configured ip mac

	[ -z "${veth}" ] && return

	configured=`nft list tables netdev | grep -w ve_${ctid}`
	if [ -z "${configured}" ]; then
		vzrun ${NFT_CMD} add table netdev ve_${ctid}
		vzrun ${NFT_CMD} add set netdev ve_${ctid} mac_list \
			'{ type ether_addr ; }'
	fi

	configured=`nft list set netdev ve_${ctid} net_class4_${class} 2> /dev/null`
	if [ -z "${configured}" ]; then
		vzrun ${NFT_CMD} add set netdev ve_${ctid} net_class4_${class} \
			'{ type ipv4_addr ; flags interval ; }'
		vzrun ${NFT_CMD} add set netdev ve_${ctid} net_class6_${class} \
			'{ type ipv6_addr ; flags interval ; }'

		class_ips=`grep "^${class} " ${VZ_IP_CLASSES} 2> /dev/null | cut -f2 -d ' '`
		for ip in ${class_ips} ; do
			[ -z "${ip##*.*}" ] && vzrun ${NFT_CMD} add element netdev \
				ve_${ctid} net_class4_${class} '{' ${ip} '}'
			[ -z "${ip##*:*}" ] && vzrun ${NFT_CMD} add element netdev \
				ve_${ctid} net_class6_${class} '{' ${ip} '}'
		done

		vzrun ${NFT_CMD} add counter netdev ve_${ctid} counter_o4_${class}
		vzrun ${NFT_CMD} add counter netdev ve_${ctid} counter_o6_${class}
		vzrun ${NFT_CMD} add counter netdev ve_${ctid} counter_i4_${class}
		vzrun ${NFT_CMD} add counter netdev ve_${ctid} counter_i6_${class}

		vzrun ${NFT_CMD} add chain netdev ve_${ctid} chain_o4_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_o4_${class} \
			counter name counter_o4_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_o4_${class} \
			meta mark set ${mark}

		vzrun ${NFT_CMD} add chain netdev ve_${ctid} chain_o6_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_o6_${class} \
			counter name counter_o6_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_o6_${class} \
			meta mark set ${mark}

		vzrun ${NFT_CMD} add chain netdev ve_${ctid} chain_i4_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_i4_${class} \
			counter name counter_i4_${class}

		vzrun ${NFT_CMD} add chain netdev ve_${ctid} chain_i6_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} chain_i6_${class} \
			counter name counter_i6_${class}
	fi

	for vdev in $veth; do
		configured=`nft list chain netdev ve_${ctid} vnet_${vdev} 2> /dev/null`
		[ -z "${configured}" ] && vzrun ${NFT_CMD} add chain netdev \
			ve_${ctid} vnet_${vdev} '{ type filter hook ingress device' \
			${vdev} 'priority 0 ; }'

		vzrun ${NFT_CMD} add rule netdev ve_${ctid} vnet_${vdev} ip \
			daddr \@net_class4_${class} jump chain_o4_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} vnet_${vdev} ip6 \
			daddr \@net_class6_${class} jump chain_o6_${class}
	done

	for mac in ${HMAC}; do
		configured=`nft list set netdev ve_${ctid} mac_list 2> /dev/null | grep -i ${mac}`
		[ -z "${configured}" ] && vzrun ${NFT_CMD} add element netdev ve_${ctid} \
			mac_list '{' ${mac} '}'
	done

	for dev in ${VZ_CONFIGURED_DEV}; do
		configured=`nft list chain netdev ve_${ctid} net_${dev} 2> /dev/null`
		if [ -z "${configured}" ]; then
			vzrun ${NFT_CMD} add chain netdev ve_${ctid} net_${dev} \
				'{ type filter hook ingress device' ${dev} \
				'priority 0 ; }'

			vzrun ${NFT_CMD} add rule netdev ve_${ctid} net_${dev} \
				'ether daddr != @mac_list return'
		fi

		vzrun ${NFT_CMD} add rule netdev ve_${ctid} net_${dev} ip \
			saddr \@net_class4_${class} jump chain_i4_${class}
		vzrun ${NFT_CMD} add rule netdev ve_${ctid} net_${dev} ip6 \
			saddr \@net_class6_${class} jump chain_i6_${class}
	done
}


# Delete nft rules for marking network packets
# Parameters:
# $1 - veth list
vzflush_nft_rules()
{
	local veth=$1
	local ctid=$(vzget_nft)
	local vdev dev mac configured

	[ -z "${veth}" ] && return

	for vdev in $veth; do
		configured=`nft list chain netdev ve_${ctid} vnet_${vdev} 2> /dev/null`
		[ -n "${configured}" ] && vzrun ${NFT_CMD} delete chain netdev ve_${ctid} vnet_${vdev}
	done

	for mac in ${HMAC}; do
		configured=`nft list set netdev ve_${ctid} mac_list 2> /dev/null | grep -i ${mac}`
		[ -n "${configured}" ] && vzrun ${NFT_CMD} delete element netdev ve_${ctid} \
			mac_list '{' ${mac} '}'
	done

	for dev in ${VZ_CONFIGURED_DEV}; do
		configured=`nft list chain netdev ve_${ctid} net_${dev} 2> /dev/null`
		[ -n "${configured}" ] && vzrun ${NFT_CMD} delete chain netdev ve_${ctid} net_${dev}
	done

	configured=`nft list tables 2>/dev/null | grep ve_${ctid}`
	[ -n "${configured}" ] && vzrun ${NFT_CMD} delete table netdev ve_${ctid}
}
