bhyve.subr

if [ ! "$_CBSD_BHYVE_SUBR" ]; then
_CBSD_BHYVE_SUBR=1
###

# generic function for bhyve
# required:
# strings.subr
#
. ${workdir}/virtual.subr

# test environment for bhyveing
init_bhyve()
{
	local _required_kmods="vmm if_tap if_bridge nmdm" _i _res

	if [ "${skip_bhyve_init_warning}" != "1" ]; then

		_res=$( /usr/bin/su -m cbsd -c ${miscdir}/popcnttest > /dev/null 2>&1 )

		if [ $? -ne 0 ]; then
			# work-around for Alex (CBSD Telegram) issue: this is not clean check for EPT existance:
			# sometimes errcode = 16 but POPCNT feature present in CPU
			# So, just print warning about this. Need for deep investigation
			${ECHO} "${MAGENTA}Warning: Seems like your CPU does not appear to support ${GREEN}EPT${NORMAL}"
			${ECHO} "${MAGENTA}But im try to run bhyve anyway...${NORMAL}"
			echo
			${ECHO} "${MAGENTA}To disable this warning, please set: ${GREEN}skip_bhyve_init_warning${MAGENTA} to: ${GREEN}1${NORMAL}"
			${ECHO} "${MAGENTA}Pause for 10 seconds${NORMAL}"
			sleep 10
		fi

		if [ "${is_virtual}" != "physical" ]; then
			${ECHO} "${MAGENTA}Warning: current environment is not physical what is required for the work of the bhyve${NORMAL}"
			${ECHO} "${MAGENTA}Virtual engine detected: ${GREEN}${is_virtual}${NORMAL}"
			${ECHO} "${MAGENTA}Please refer to this page for more information: ${GREEN}https://wiki.freebsd.org/bhyve${NORMAL}"
			${ECHO} "${MAGENTA}Try to run bhyve anyway...${NORMAL}"
			${ECHO} "${MAGENTA}To disable this warning, please set: ${GREEN}skip_bhyve_init_warning${MAGENTA} to: ${GREEN}1${NORMAL}"
			${ECHO} "${MAGENTA}Pause for 10 seconds${NORMAL}"
			sleep 10
		fi
	fi

	for _i in ${_required_kmods}; do
		if ! /sbin/kldstat -qm ${_i} >/dev/null 2>&1; then
			${ECHO} "${MAGENTA}No kldloaded module: ${GREEN}${_i}${NORMAL}"
			${ECHO} "${MAGENTA}Please add ${GREEN}vmm_load=\"YES\"${MAGENTA} into /boot/loader.conf and${MAGENTA}"
			${ECHO} "${MAGENTA}put ${GREEN}kld_list=\"if_bridge if_tap nmdm\"${MAGENTA} into your ${GREEN}/etc/rc.conf${MAGENTA} then reboot host.${NORMAL}"
			[ -n "${DIALOG}" ] && ${ECHO} "${MAGENTA}Press any key...${NORMAL}" && read p

			# CBSD QUEUE
			if [ -x "${moduledir}/cbsd_queue.d/cbsd_queue" ]; then
				[ "${cbsd_queue_name}" != "none" ] && cbsd_queue cbsd_queue_name=${cbsd_queue_name} id=${jname} cmd=bstart status=2 data_status=1
			fi

			exit 0
		fi
	done

	tmuxcmd=$(which tmux)
	[ -z "${tmuxcmd}" ] && err 1 "${MAGENTA}Current version required for ${GREEN}tmux${MAGENTA}\nPlease ${GREEN}pkg install tmux ${MAGENTA} or ${GREEN}make -C /usr/ports/sysutils/tmux install${MAGENTA} it.${NORMAL}"
}


# autoincement for $bhyve_pci_index variable which symbolizes pci id bus
next_pci_id()
{
	[ -z "${bhyve_pci_index}" ] && bhyve_pci_index="0"
	bhyve_pci_index=$(( bhyve_pci_index + 1 ))
	[ ${bhyve_pci_index} -gt 31 ] && ${ECHO} "${MAGENTA}No free available bhyve_pci_index, 31 is max${NORMAL}" && return 1
	return 0
}

# autoincement for nmdm_index
next_nmdm_id()
{
	local _i
	nmdm_index=0

	for _i in $( /usr/bin/seq 0 500 ); do
		/bin/ls /dev/nmdm${_i}A* > /dev/null 2>&1
		if [ $? -ne 0 ]; then
			nmdm_index=${_i}
			return 0
		fi
	done

	[ ${_i} -eq 500 ] && return 1
	return 0
}

compile_uefi_boot_args()
{
	if ! next_pci_id; then
		# no free pci bus
		exit 0
	fi

	#also same string in compile_cd_args cd_args
	case "${vm_boot}" in
		hdd)
			if [ "${vm_os_type}" != "openbsd" -a "${vm_os_type}" != "freebsd" ]; then
				uefi_boot_args="-s ${bhyve_pci_index}:0,ahci-cd,/usr/local/cbsd/upgrade/patch/efirefd.fd"
			fi
			;;
		net)
			uefi_boot_args="-s ${bhyve_pci_index}:0,ahci-cd,/usr/local/cbsd/upgrade/patch/efirefn.fd"
			;;
	esac
}


# export arguments for bhyve about dsk vms in $dsk_args variable
# $jname must be set's
# sample:
# jname="debian"
# if compile_dsk_args; then
# 	echo $dsk_args
# else
#	echo "No disks"
# fi
compile_dsk_args()
{
	local dsk_id=0
	local sqldelimer=" "
	local prefix
	local full_dsk_path
	local _shared_devs

	# temporary workaroung for Alex (CBSD Telegram) issue with no disk (old SQL schema) - don't select dsk_conf
	#	eval $( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT dsk_controller,dsk_path,dsk_slot,dsk_conf FROM bhyvedsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" |while read dsk_controller dsk_path dsk_slot dsk_conf; do
	eval $( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT dsk_controller,dsk_path,dsk_slot FROM bhyvedsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" |while read dsk_controller dsk_path dsk_slot; do

		if ! next_pci_id; then
			# no free pci bus
			exit 0
		fi

		if [ ${dsk_slot} -eq 0 ]; then
			pcislot=${bhyve_pci_index}
		else
			pcislot=${dsk_slot}
		fi

		dsk_conf=$( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT dsk_conf FROM bhyvedsk WHERE jname=\"${jname}\" AND dsk_path=\"${dsk_path}\" LIMIT 1 2>/dev/null )

		# test for full path. If path not started from '/' - append $data dir
		full_dsk_path=
		prefix=$( substr --pos=0 --len=1 --str="${dsk_path}" )

		if [ "${prefix}" != "/" ]; then
			full_dsk_path="${data}/${dsk_path}"
		else
			full_dsk_path="${dsk_path}"
		fi

		if [ -n "${dsk_conf}" ]; then
			echo "export dsk${dsk_id}=\"-s ${pcislot}:0,${dsk_controller},${full_dsk_path},${dsk_conf}\" ;"
		else
			echo "export dsk${dsk_id}=\"-s ${pcislot}:0,${dsk_controller},${full_dsk_path}\" ;"
		fi

		dsk_id=$(( dsk_id + 1 ))
	done ) || err 1 "${MAGENTA}Error while create disk map${NORMAL}"

	dsk_args=""
	for i in $( /usr/bin/seq 0 31 ); do
		eval T="\$dsk$i"
		[ -z "${T}" ] && break
		# make custom argument for bhyve and add count to bhive_pci_index cause increment in while loop we lose
		if ! next_pci_id; then
			# no free pci bus
		fi
		dsk_args="${dsk_args} ${T}"
	done

	# shared devices
	_shared_devs=$( cbsdsql storage_media "SELECT path FROM media WHERE type=\"shared\" AND jname=\"${jname}\"" )

	for i in ${_shared_devs}; do
		if ! next_pci_id; then
			# no free pci bus
		fi
		dsk_args="${dsk_args} -s ${bhyve_pci_index}:0,virtio-blk,${i}"
	done

	if [ -r ${data}/cbsd.img ]; then
		if ! next_pci_id; then
			# no free pci bus
		fi
		dsk_args="${dsk_args} -s ${bhyve_pci_index}:0,virtio-blk,${data}/cbsd.img"
	fi

	[ -z "${dsk_args}" ] && return 1
	return 0
}

# print 5c:f9:dd:76:d5:c4 from 5cf9dd76d5c4
normalize_nic_hwaddr()
{
	local _hwaddr
	[ -z "${1}" ] && return 0

	_hwaddr=$( echo "${1}" |/usr/bin/tr -d ":"| /usr/bin/sed -e 's/\([0-9A-Fa-f]\{2\}\)/\1:/g' -e 's/\(.*\):$/\1/' )
	printf ${_hwaddr}
}


# export arguments for bhyve about nic vms in $nic_args variable
# $jname must be set's
# sample:
# jname="debian"
# if compile_nic_args; then
# 	echo $nic_args
# else
#	echo "No nic"
# fi
compile_nic_args()
{
	local _id=0 i _res taplist=
	local sqldelimer=" "
	local mybridge
	local net_emul=
	local _parent_mtu
	local _ret=
	local _is_bridge=
	local autoconn_iface=$( /usr/sbin/sysrc -n cbsd_autoconnect_bridge 2>&1 )
	. ${workdir}/vnet.subr # get_vm_uplink_interface

	local errmsg=

	eval $( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT id,nic_order,nic_driver,nic_slot,nic_type,nic_parent,nic_hwaddr,nic_address,nic_mtu FROM bhyvenic WHERE jname=\"${jname}\" |while read nic_id nic_order nic_driver nic_slot nic_type nic_parent nic_hwaddr nic_address nic_mtu; do

		if ! next_pci_id; then
			# no free pci bus
			exit 0
		fi

		if [ ${nic_slot} -eq 0 ]; then
			pcislot=${bhyve_pci_index}
		else
			pcislot=${nic_slot}
		fi

		case "${nic_driver}" in
			e1000)
				net_emul="e1000"
				;;
			*)
				net_emul="virtio-net"
				;;
			esac

		case ${nic_parent} in
			vale_*)
				get_vm_uplink_interface ${nic_parent}
				;;
			*)
				_is_bridge=$( substr --pos=0 --len=6 --str=${nic_parent} )
				if [ "${_is_bridge}" != "bridge" ]; then
					# this is not bridge, detect uplink iface
					get_vm_uplink_interface ${nic_parent}

					if [ "${interface}" != "disable" ]; then
						# always up parent device
						/sbin/ifconfig ${interface} up
						if ! mybridge=$( get_my_device bridge ${interface} ); then
							err 1 "errmsg=\"Error:get_my_device_bridge_for_${interface}: ${mybridge}\""
						fi
					else
						# dont create/attach to bridge
						mybridge="disable"
					fi
				else
					/sbin/ifconfig ${nic_parent} > /dev/null 2>&1
					_ret=$?
					[ ${_ret} -ne 0 ] && err 1 "errmsg=\"Error:no_such_bridge: ${nic_parent}. Please create it\""
					mybridge="${nic_parent}"
				fi

				mytap=$( get_my_tap ${mybridge} )
				_ret=$?
				[ ${_ret} -eq 1 ] && err 1 "errmsg=\"Error:get_my_tap_for_${mybridge}: ${mybridge}\""

				/sbin/ifconfig ${mytap} description ${jname}-nic${_id} up
				if [ "${mybridge}" != "disable" ]; then
					if [ "${autoconn_iface}" = "NO" ]; then
						/sbin/ifconfig ${mybridge} addm ${mytap} >/dev/null 2>&1
					else
						/sbin/ifconfig ${mybridge} addm ${interface} addm ${mytap} >/dev/null 2>&1
					fi
				fi

				if [ -n "${nic_address}" ]; then
					if [ "${nic_address}" != "0" -o "${nic_address}" != "disable" ]; then

						IFS=","
						for ip in ${nic_address}; do
							if [ "${ip}" = "DHCP" ]; then
								ip=$( dhcpd )
								[ $? -eq 2 ] && err 1 "${MAGENTA}No free IP address for DHCP in nodeippool${NORMAL}"
							fi
							IFS=" "
							ipwmask ${ip}
							iptype ${IWM}
							local _inet=$?
							case "${_inet}" in
								1)
									local proto="inet"
									;;
								2)
									local proto="inet6"
									;;
								*)
									continue
									;;
							esac
							# ipv type + mask
							/sbin/ifconfig ${mytap} ${proto} ${ip} alias
						IFS=","
					done
					IFS=" "
					fi
				fi

				;;
		esac

		# MAC MGMT
		if [ "${nic_hwaddr}" = "0" ]; then
			nic_hwaddr=$( mac_gen 00:a0:98 )
			cbsdsql ${jailsysdir}/${jname}/local.sqlite "UPDATE bhyvenic SET nic_hwaddr=\"${nic_hwaddr}\" WHERE id=\"${nic_id}\""
		fi

		if [ "${nic_hwaddr}" != "0" ]; then
			unset mymac
			mymac=$( normalize_nic_hwaddr ${nic_hwaddr} )
			echo "export nic${_id}=\"-s ${pcislot}:0,${net_emul},${mytap},mac=${mymac}\" ;"
		else
			echo "export nic${_id}=\"-s ${pcislot}:0,${net_emul},${mytap}\" ;"
		fi

		# MTU management
		[ -z "${nic_mtu}" ] && nic_mtu="0"	# AUTO
		if [ ${nic_mtu} -eq 0 ]; then
			# Get parent MTU size
			_parent_mtu=$( ${toolsdir}/nic_info --nic=${mybridge} --quiet 2>/dev/null )
			_ret=$?
			if [ ${_ret} -eq 0 ]; then
				[ -n "${_parent_mtu}" ] && _ret=$( /sbin/ifconfig ${mytap} mtu ${_parent_mtu} )
			fi
		else
			# Get parent MTU size
			# compare MTU with parent
			#_parent_mtu=$( ${toolsdir}/nic_info --nic=${mybridge} --quiet 2>/dev/null )
			#_ret=$?
			#if [ ${_ret} -eq 0 ]; then
			#	[ ${nic_mtu} -gt ${_parent_mtu} ]
			#fi
			_ret=$( /sbin/ifconfig ${mytap} mtu ${nic_mtu} )
		fi

		echo "export mytap${_id}=\"${mytap}\";"

		_id=$(( _id + 1 ))
	done ) || err 1 "${MAGENTA}Error while create nic map: ${errmsg}${NORMAL}"

	[ -n "${errmsg}" ] && err 1 "${MAGENTA}Error while create nic map: ${GREEN}${errmsg}${NORMAL}"

	nic_args=""

	# export in mytap fill list of vm NICs
	mytap=""

	for i in $( /usr/bin/seq 0 31 ); do
		eval T="\$nic$i"
		eval taplist="\$mytap$i"
		[ -z "${T}" ] && break
		[ -z "${taplist}" ] && break
		# make custom argument for bhyve and add count to bhive_pci_index cause increment in while loop we lose
		if ! next_pci_id; then
			# no free pci bus
		fi
		nic_args="${nic_args} ${T}"
		mytap="${mytap} ${taplist}"
	done

	[ -z "${nic_args}" ] && return 1

	return 0
}


# export arguments for bhyve hostbridge_args
compile_hostbridge_args()
{
	# This is usually configured at slot 0

	# what the diffrence?
	#hostbridge_args="-s 0,${vm_hostbridge}"
	hostbridge_args="-s 0:0,${vm_hostbridge}"
}

# export arguments for bhyve lpc_args
compile_lpc_args()
{
#	if ! next_pci_id; then
#		return 1
#	fi

	lpc_args="-s 31,lpc"
}

# export arguments for bhyve_cpus
compile_bhyve_cpus_args()
{
	local _sockets _cores _threads _query
	bhyve_cpus="${vm_cpus}"

	# waiting for https://reviews.freebsd.org/D9930
	#return 0

	# no topology when single core or vm_cpu_topology empty
	[ "${vm_cpus}" = "1" -o -z "${vm_cpu_topology}" ] && return 0

	# not configured
	case "${vm_cpu_topology}" in
		0|default)
			return 0
			;;
	esac

	_query=$( cbsdsql local SELECT sockets,cores,threads FROM vm_cpu_topology WHERE name=\"${vm_cpu_topology}\" )
	if [ -z "${_query}" ]; then
		${ECHO} "${MAGENTA}Warning: no vm_cpu_topology: ${GREEN}${vm_cpu_topology}${MAGENTA}. Skip.${NORMAL}"
		return 0
	fi

	OIFS="${IFS}"
	IFS="|"
	sqllist "${_query}" _sockets _cores _threads
	IFS="${OIFS}"
	if [ -z "${_sockets}" ]; then
		${ECHO} "${MAGENTA}Warning: vm_cpu_topology ${vm_cpu_topology}: Empty ${GREEN}sockets${MAGENTA}. Skip.${NORMAL}"
		return 0
	fi
	if [ -z "${_cores}" ]; then
		${ECHO} "${MAGENTA}Warning: vm_cpu_topology ${vm_cpu_topology}: Empty ${GREEN}cores${MAGENTA}. Skip.${NORMAL}"
		return 0
	fi
	if [ -z "${_threads}" ]; then
		${ECHO} "${MAGENTA}Warning: vm_cpu_topology ${vm_cpu_topology}: Empty ${GREEN}threads${MAGENTA}. Skip.${NORMAL}"
		return 0
	fi

	vm_cpus=$( get_vm_cores_by_topology ${_sockets} ${_cores} ${_threads} )

	if [ -z "${vm_cpus}" ]; then
		bhyve_cpus="1"
	else
		bhyve_cpus="cpus=${vm_cpus},sockets=${_sockets},cores=${_cores},threads=${_threads}"
	fi
	return 0
}

# export arguments for bhyve pci_passthru_args
compile_pci_passthru_args()
{
	local ppt _i

	ppt=$( cbsdsql local SELECT ppt FROM bhyveppt WHERE jname=\"${jname}\" 2>/dev/null )

	[ -z "${ppt}" ] && return 0

	pci_passthru_args=

	for _i in ${ppt}; do

		if ! next_pci_id; then
			return 1
		fi

		${ECHO} "${MAGENTA}Passthru enabled: ${GREEN}${_i}${NORMAL}"

		pci_passthru_args="${pci_passthru_args} -s ${bhyve_pci_index},passthru,${_i}"
	done

	pci_passthru_args="${pci_passthru_args}"

	. ${workdir}/virtual.subr

	if check_dmar; then
		err 1 "${MAGENTA}I/O MMU / VT-d not enabled. Check you hardware or BIOS setting${NORMAL}"
	fi

	return 0
}


# export arguments for bhyve efi_args
compile_efi_args()
{
	case "${vm_efi}" in
		uefi)
			efi_args="-l bootrom,${efi_firmware}"
			;;
		uefi_csm)
			efi_args="-l bootrom,${efi_firmware_csm}"
			;;
		*)
			efi_args=""
			;;
	esac

	[ -z "${efi_args}" ] && return 0

	return 0
}

# export arguments for bhyve console_args
compile_console_args()
{
	local com1=0 com2=0

	case "${vm_console}" in
		"nmdm")

			if ! next_nmdm_id ; then
				${ECHO} "${MAGENTA}No such next_nmdm_id for VMs: ${GREEN}${jname}${NORMAL}"
				return 1
			fi

			com1=${nmdm_index}
			com2=$(( nmdm_index + 1 ))

			console_args="-l com1,/dev/nmdm${com1}A -l com2,/dev/nmdm${com2}A"
			console_nmdm="/dev/nmdm${com1}B /dev/nmdm${com2}B"
			;;
		*)
			console_args="-l com1,stdio"
			;;
	esac
	return 0
}

# export arguments for bhyve virtio-rnd
compile_virtiornd_args()
{
	# this is feature of FreeBSD 10.1+
	[ ${freebsdhostversion} -lt 1000730 ] && return 0

	if ! next_pci_id; then
		return 1
	fi

	virtiornd_args="-s ${bhyve_pci_index}:0,virtio-rnd"
}

# export arguments for bhyve virtio-9p
compile_virtio_9p_args()
{
	# this is feature of FreeBSD 12.+
	[ ${freebsdhostversion} -lt 1000730 ] && return 0
	local shares_num
	local mydb
	local last_share

	shares_num=1
	mydb="${jailsysdir}/${jname}/local.sqlite"

	sqldelimer=" "

	eval $( cbsdsql ${mydb} SELECT p9path,p9device FROM p9shares 2>/dev/null | while read _p9path _p9device; do
		echo "p9path${shares_num}=\"${_p9path}\""
		echo "p9device${shares_num}=\"${_p9device}\""
		shares_num=$(( shares_num + 1 ))
	done )

	for last_share in $( /usr/bin/seq 1 24 ); do
		unset p9path p9device
		eval p9path="\$p9path$last_share"
		[ -z "${p9path}" ] && break
		eval p9device="\$p9device$last_share"

		unset \$p9path$last_share
		unset \$p9device$last_share

		if ! next_pci_id; then
			return 1
		fi

		if [ -d ${p9path} ]; then
			${ECHO} "${MAGENTA}virtio_p9_shares: shared directory enabled: ${LYELLOW}${p9device} ${MAGENTA} ${MAGENTA}-> ${p9path}${NORMAL}"
			${ECHO} "  ${MAGENTA}Use to mount: mount -t 9p -o trans=virtio ${p9device} /mnt"
			virtio_9p_args="${virtio_9p_args} -s ${bhyve_pci_index},virtio-9p,${p9device}=${p9path}"
		else
			${ECHO} "${MAGENTA}virtio_p9_shares error: directory doesn't exist, skipp for share: ${GREEN}${p9path}${NORMAL}"
		fi
	done

	return 0
}

# export arguments for bhyve cd_args and cd_args2 (second boot after empty disk check)
# $iso_img must be present
compile_cd_args()
{
	cd_args=
	cd_args2=

	# Ubuntu has trouble when got two CD
	#[ "${vm_boot}" = "net" ] && return 0

	if ! next_pci_id; then
		return 1
	fi

	if [ -z "${iso_img}" ]; then
		${ECHO} "${MAGENTA}iso_img variable is not set${NORMAL}"
		return 1
	fi

	if [ ! -r "${iso_img}" ]; then
		iso_img2=$( cbsdsql storage_media SELECT path FROM media WHERE jname=\"${jname}\" AND type=\"iso\" LIMIT 1 )
		if [ ! -r ${iso_img2} ]; then
			${ECHO} "${MAGENTA}iso_img in not readable: ${GREEN}${iso_img} ${iso_img2}${NORMAL}"
			return 1
		else
			iso_img="${iso_img2}"
		fi
	fi

	if [ -z "${iso_img}" ]; then
		${ECHO} "${MAGENTA}iso_img variable is empty: no ISO or image is not readable${NORMAL}"
		return 1
	fi

	## for check via ref
	# cd_args="-s ${bhyve_pci_index}:0,ahci-cd,${distdir}/upgrade/patch/efirefd.fd"
	#if ! next_pci_id; then
	#	return 1
	#fi
	# cd_args="${cd_args} -s ${bhyve_pci_index}:0,ahci-cd,${iso_img}"

	if [ "${vm_os_type}" = "openbsd" ]; then
		# small hack: openbsd use install.fs as hdd drive, not cd
		cd_args="-s ${bhyve_pci_index}:0,ahci-hd,${iso_img},ro"
		cd_args2=""
	else
		cd_args="-s ${bhyve_pci_index}:0,ahci-cd,${iso_img}"
		case ${vm_os_type} in
			freebsd)
				# Latest FreeBSD has troubles with UIEF boot via refind, use default firmware
				cd_args2=""
				;;
			*)
				cd_args2="-s ${bhyve_pci_index}:0,ahci-cd,/usr/local/cbsd/upgrade/patch/efirefd.fd"
				;;
		esac
	fi
}

# create new disk image for bhyve VMs
# -f fsfeat ( 1 or 0 ) - use of skip FS features
# -p full path
# -s size (eg: 1g)
# -t type (zvol, md)
# require: /usr/local/bin/cbsd as shell
# require: ${tools}
#
# if ! bhyve_create_dsk -p /tmp/olala.vhd -s 10g -f 0; then
#    echo ";=("
# fi
bhyve_create_dsk()
{
	local newdsk imgsize fsfeat imgtype _msg

	while getopts "f:p:s:t:" opt; do
		case "${opt}" in
			f) feat="${OPTARG}" ;;
			p) newdsk="${OPTARG}" ;;
			s) imgsize="${OPTARG}" ;;
			t) imgtype="${OPTARG}" ;;
		esac
		shift $(($OPTIND - 1))
	done

	local _res

	if [ "${fsfeat}" = "0" ]; then
		zfsfeat=0
		hammerfeat=0
	fi

	local lunname=$( /usr/bin/basename ${newdsk} ) # for ZFS vol
	local datadir=$( /usr/bin/dirname ${newdsk} )  # for ZFS vol

	local convval imgbytes blockcount

	if [ -f "${newdsk}" ]; then
		echo "disk ${newdsk} already exist"
		return 1
	fi

	# test for human
	if is_number ${imgsize}; then
		conv2bytes ${imgsize} || err 1 "${MAGENTA}conv2bytes from bhyve.subr error for: ${GREEN}${imgsize}${NORMAL}"
		imgbytes="${convval}"
	else
		imgbytes=${imgsize}
	fi

	[ ! -d "${datadir}" ] && /bin/mkdir -p ${datadir}
	if [ -z "${imgtype}" ]; then
		case ${zfsfeat} in
			1)
				imgtype="zvol"
				;;
			*)
				imgtype="md"
				;;
		esac
	fi

	if [ "${zfsfeat}" = "1" -a "${imgtype}" = "zvol" ]; then
		readconf zfs.conf
		. ${zfstool}

		DATA=$( /sbin/zfs get -Ho value name ${jaildatadir} )
#		ZPOOL=$( /sbin/zfs get -Ho value name ${jaildatadir} )
#		if zfsroot $jname; then
#			err 1 "$ZPOOL/$jname already in use"
#		fi
#		/sbin/zfs create ${zfs_create_flags} -o mountpoint=${JAILDIR} ${ZPOOL}/${jname}
		# zfs_create_sparse defined via zfs.conf
		if [ "${zfs_create_sparse}" = "0" ]; then
			_msg=$( /sbin/zfs create -V ${imgbytes} -o volmode=dev ${DATA}/bcbsd-${jname}-${lunname} 2>&1 )
			_res=$?
		else
			_msg=$( /sbin/zfs create -V ${imgbytes} -s -o volmode=dev ${DATA}/bcbsd-${jname}-${lunname} 2>&1 )
			_res=$?
		fi

		if [ ${_res} -ne 0 ]; then
			echo "${_msg}"
			return ${_res}
		fi

		/bin/ln -sf /dev/zvol/${DATA}/bcbsd-${jname}-${lunname} ${newdsk}

	elif [ "${hammerfeat}" = "1" ]; then
		err 1 "Hammer not supported yet"
	else
		blockcount=$(( imgbytes  / 1048576 ))
		[ ! -f "${newdsk}" -a ! -h "${newdsk}" ] && /usr/bin/touch "${newdsk}"
		/bin/dd if=/dev/zero of="${newdsk}" bs=1m count=0 seek=${blockcount} 1> /dev/null 2>&1 || return 1
	#	/bin/dd if=/dev/random of="${newdsk}" bs=1m count=${blockcount}
		_res=$?
		[ ${_res} -ne 0 ] && return ${_res}
	fi

	media mode=register name="hdd-${lunname}" path="${newdsk}" type="hdd" jname="${jname}"
	return 0
}

# detach and remove disk image from bhyve VMs
# jname must be set
# $1 dsk_path from sql table (eg: dsk1.vhd )
# require: /usr/local/bin/cbsd as shell
bhyve_remove_dsk()
{
	local dsk="${1}"; shift
	[ -z "${jname}" -o -z "${dsk}" ] && return 0

	[ -r ${jailsysdir}/${jname}/local.sqlite ] && cbsdsql ${jailsysdir}/${jname}/local.sqlite DELETE FROM bhyvedsk WHERE dsk_path=\"${dsk}\" AND jname=\"${jname}\"

	if [ "${zfsfeat}" = "1" ]; then
		readconf zfs.conf
		. ${zfstool}

		if is_getzvol ${data}/${dsk}; then
			/sbin/zfs destroy ${is_zvol}
		fi

	elif [ "${hammerfeat}" = "1" ]; then
		err 1 "Hammer not supported yet"
	else

	fi

	[ -f "${data}/${dsk}" -o -h "${data}/${dsk}" ] && /bin/rm -f "${data}/${dsk}"

	return 0
}

# detach and remove NIC from bhyve VMs
# jname must be set
# $1 is id of nic in SQL table
# require: /usr/local/bin/cbsd as shell
bhyve_remove_nic()
{
	local id="${1}"; shift
	[ -z "${jname}" -o -z "${id}" ] && return 0

	cbsdsql ${jailsysdir}/${jname}/local.sqlite DELETE FROM bhyvenic WHERE id=\"${id}\" AND jname=\"${jname}\"

	return 0
}


compile_vnc_args()
{
	local vm_port
	local vnc_bind

	local width height

	vnc_args=

	if [ ${freebsdhostversion} -lt 1100120 ]; then
		${ECHO} "${MAGENTA}VNC available only on FreeBSD 11.0+. Disable VNC${MAGENTA}"
		sleep 2
		return 0
	fi

	case "${vm_vnc_port}" in
		0)
			vm_port=$( get_next_rd_port )
			;;
		1)
			return 1
			;;
		*)
			vm_port="${vm_vnc_port}"
			;;
	esac

	if ! next_pci_id; then
		return 1
	fi

	echo "${vm_port}" > ${jailsysdir}/${jname}/vnc_port

	if [ -n "${bhyve_vnc_tcp_bind}" ]; then
		vnc_bind="${bhyve_vnc_tcp_bind}"
	else
		vnc_bind="${default_vnc_tcp_bind}"
	fi

	# add bhyve_vnc_tcp_ipconnect variables, used in
	# vncviewer promt and exported hooks variables
	if [ "${vnc_bind}" = "0.0.0.0" ]; then
		bhyve_vnc_tcp_ipconnect="${nodeip}"
	else
		bhyve_vnc_tcp_ipconnect="${vnc_bind}"
	fi

	if [ -n "${bhyve_vnc_resolution}" ]; then
		width=${bhyve_vnc_resolution%%x*}
		height=${bhyve_vnc_resolution##*x}
	else
		width="${default_vnc_width}"
		heigh="${default_vnc_height}"
	fi

	vnc_args="-s ${bhyve_pci_index},fbuf,tcp=${vnc_bind}:${vm_port},w=${width},h=${height}"

	if [ "${vm_efi}" != "none" ]; then
		${ECHO} "${MAGENTA}VRDP is enabled. VNC bind/port: ${GREEN}${bhyve_vnc_tcp_ipconnect}:${vm_port}${NORMAL}"
		${ECHO} "${MAGENTA}For attach VM console, use: ${GREEN}vncviewer ${bhyve_vnc_tcp_ipconnect}:${vm_port}${NORMAL}"
		${ECHO} "${MAGENTA}Resolution: ${GREEN}${width}x${height}.${NORMAL}"
		[ -n "${vnc_password}" ] && ${ECHO} "${MAGENTA}VNC pass: ${GREEN}${vnc_password}${NORMAL}"

		if [ "${vnc_bind}" = "0.0.0.0" ]; then
			echo
			${ECHO} "${MAGENTA}Warning!!! You are running system with open ${GREEN}VNC${MAGENTA} port to all world wich is not to safe${MAGENTA}"
			${ECHO} "${MAGENTA}Please use IP filter or balancer with password to restrict ${GREEN}VNC${MAGENTA} port access${NORMAL}"
			${ECHO} "${MAGENTA}Or change vnc_bind params to ${GREEN}127.0.0.1${MAGENTA} and reboot VM after maintenance work${NORMAL}"
			echo
		fi
	fi

	return 0
}

###
fi