Skip to content

Commit

Permalink
Cleanup, extra safety nets and Looking Glass support for multi-gpu ho…
Browse files Browse the repository at this point in the history 
…sts.
  • Loading branch information
Jared J committed Jun 6, 2021
1 parent 8ef30e1 commit e649cb6
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 37 deletions.
5 changes: 5 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ This example would catch any:

Prints IOMMU groupings if available then exists.

`-looking-glass` / `-lookingglass` / `-lg`

Adds a 64MB shared memory module for the Looking Glass project and a spice server onto the guest for input from host during Looking Glass usage.
You will still need to go into your VM add the VirtIO IVSHMEM driver to the "PCI standard RAM Controller" which appears in Device Manager under System Devices before Looking Glass will function.

`-extras '-device xyz -device abc -device ac97 -display gtk -curses'`

If set adds extra arbitrary commands to the cmdline of qemu (Once invoked)
Expand Down
152 changes: 115 additions & 37 deletions main
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ function printHelp {

echo -e "[Required for most use-cases]"
echo -e " ${colors[cyan]}-image /dev/zvol/zpool/windows${colors[none]}"
echo -e " ${colors[cyan]}-imageformat${colors[none]}/-format raw/qcow/vdi/etc${colors[none]}"
echo -e " ${colors[cyan]}-imageformat${colors[none]}/${colors[cyan]}-format${colors[none]} raw/qcow/vdi/etc${colors[none]}"
echo -e " A file or blockdevice to give the guest. Complete with it's [Qemu recognised] format\n"

echo -e "\n[For first-install or liveCD booting]"
Expand All @@ -62,7 +62,7 @@ function printHelp {

echo -e " ${colors[cyan]}-bridge br0,tap0${colors[none]}\t(Attach vm's tap0 to existing br0)"
echo -e " ${colors[cyan]}-bridge br0,tap0,eth0${colors[none]}\t(Create br0, attach vm's tap0 and host's eth0 to br0, use dhclient for a host IP.)"
echo -e " If no network options are specified the default qemu NAT adapter will be used."
echo -e " If no network options are specified the default qemu NAT adapter will be used.\n"

echo -e " ${colors[cyan]}-nonet${colors[none]}"
echo -e " If specified, the VM is not given a network adapter. Useful if you're passing a USB or PCI network card.\n"
Expand All @@ -76,14 +76,20 @@ function printHelp {

echo -e " ${colors[cyan]}-hyperv${colors[none]}"
echo -e " Enable hyper-v enlightenments for nested virtualization."
echo -e " Can help invasive anticheats play along however HyperV will need to be enabled in the guest's Features"
echo -e " Can help invasive anticheats play along however HyperV will need to be enabled in the guest's Features.\n"

echo -e " ${colors[cyan]}-usb 'SteelSeries|Keyboard|Xbox|1234:5678'${colors[none]}"
echo -e " A regex for lsusb parsing. Can be anything that matches a specific device line\n"
echo -e " A regex for lsusb parsing. Can be anything that matches a specific device line.\n"

echo -e " ${colors[cyan]}-pci 'Realtek|NVIDIA|10ec:8168'${colors[none]}"
echo -e " A regex for lspci parsing. Example will take any nvidia and Realtek cards, plus any device with the ID.\n"

echo -e " ${colors[cyan]}-looking-glass ${colors[none]}/${colors[cyan]} -lookingglass ${colors[none]}/${colors[cyan]} -lg ${colors[none]}"
echo -e " Adds a 64MB shared memory module for the Looking Glass project and a spice server onto the guest for input from host during Looking Glass usage.\n"

echo -e " ${colors[cyan]}-extras '-device abc123,b,c,d=e' / -extras '-device AC97' ${colors[none]}"
echo -e " Accepts a string for arbitrary extra qemu arguments. Highly useful when I want to use extra qemu features without having to implement them into the script right away.\n"

echo -e " ${colors[cyan]}-taskset 0,1,2,3,4,5,6 / ${colors[cyan]}-taskset 0,2,4,8${colors[none]}"
echo -e " A comma delimited list of host threads the Guest is allowed to execute on."
echo -e " (Setting this also shapes the guest CPU topology to match.)"
Expand Down Expand Up @@ -128,8 +134,17 @@ function isDry { if [ $DRY -eq 1 ] ; then return 0 ; else return 1 ; fi ; }
# Read numeric value from meminfo
function readMeminfo { if [ ! -z "$1" ]; then grep -m1 "$1" /proc/meminfo | grep -Eo '[0-9]+'; fi ; }

# Try to undo any mess we've caused
function gpuCheck { if grep -o '\[VGA controller\]' <<<$(lspci -vnnn -s "$1") > /dev/null 2>&1 ; then return 0 ; else return 1 ; fi }

function bindingTimeout {
if [ ! -z "$1" ]; then attemptedTask=$1 ; else attemptedTask="bind or unbind" ; fi
echo -e "${colors[red]} The device ${colors[none]} $fullBuspath ${colors[red]}// ${colors[none]}$vendorClass ${colors[red]} Was unable to $attemptedTask after 5 seconds, is something else using it? (E.g This will happen to a GPU in use by X). Giving up.${colors[none]}"
exit 1
}

# Try to undo any mess we've caused at the end of a run
function do_cleanup {
echo "Cleaning up.."
permissionsManager return # Restore original permissions if any
echo -ne "${colors[none]}"
if ! isDry
Expand All @@ -152,7 +167,9 @@ function do_cleanup {
if [ "$hugeMount" == "self" ]; then umount $hugePath ; fi #umount if we mounted it
fi
fi
[ -f '/dev/shm/looking-glass' ] && rm /dev/shm/looking-glass
echo -e "\n${colors[green]}Cleanup complete.${colors[none]}"
exit
}

function regexCleanup {
Expand Down Expand Up @@ -213,7 +230,7 @@ function bridger {
then
if [ $bridgePreexists -eq 0 ]
then
echo -e "${colors[yellow]}Cleaning up our bridge..."
echo -e "${colors[yellow]}Undoing our bridge..."
dhclient -r $br >/dev/null 2>&1 && echo -e "${colors[green]}dhcp lease released and dhclient process ended...${colors[none]}"
for i in $tap $br; do sudo ip link set $i down ; sudo ip link del $i; done # Remove the tap and bridge
if ! ip link show br0 >/dev/null 2>&1; then echo -e "${colors[green]}Bridge removed.${colors[none]}"; fi
Expand Down Expand Up @@ -261,32 +278,37 @@ function enumerateUSBs { # This only makes arguments

function enumeratePCIs { # This makes arguments and also unbinds from drivers. Optional vfio-pci rebinding.
function gpuDetectedAndInUse {
echo -e "${colors[yellow]}Video device $1 appears to be in use by the host (Display-manager, efi-framebuffer[virtual consoles] or otherwise).${colors[none]}"
echo -e "${colors[yellow]}At the current time I'll need to stop your display manager to continue.${colors[none]}"
echo -e "${colors[yellow]}If you don't think the host is using this [RIGHT NOW] then there may be an error.${colors[none]}"
echo -e "${colors[yellow]} Video device $1 is bound to a driver which isn't vfio-pci and could be in use by the DM, framebuffer or otherwise.${colors[none]}"
echo -e "${colors[yellow]} For this reason the script will now attempt to stop the display-manager service and unbind "
echo -e "${colors[yellow]} the efi framebuffer instead of risking a driver unbind deadlock in waiting for X to quit.${colors[none]}"
echo -e "${colors[yellow]} If your X server and virtual consoles don't use this card you can unbind it from its driver manually before running this script.${colors[none]}"
if systemctl is-active display-manager >/dev/null && ! isDry
then
echo -e "${colors[yellow]}Stopping display-manager and unbinding console drivers in 5 seconds...${colors[none]}" ; sleep 5
echo -e "${colors[yellow]} Stopping display-manager and unbinding console drivers in 5 seconds...${colors[none]}" ; sleep 5
sudo systemctl stop display-manager ; export dm="seen"
else
echo -e "${colors[yellow]}Could not find a service for display-manager.${colors[none]}"
echo -e "${colors[yellow]} Could not find a service for display-manager. Confused but trying to continue anyway.${colors[none]}"
fi
consoleDrivers unbind && echo -e "${colors[yellow]} Console drivers unbound, host may be headless"
consoleDrivers unbind
}

echo "PCI:"
declare -A -g deviceDrivers # Array for tracking devices and their driver.
if ! lsmod|grep -q vfio_pci; then sudo modprobe vfio-pci; fi
if [ "$?" -ne "0" ]; then echo "Could not modprobe vfio-pci. Please fix this."; exit 1 ; fi
if ! lsmod|grep -q vfio_pci;
then
echo -e "${colors[yellow]} vfio-pci isn't loaded. Loading it now.${colors[none]}"
sudo modprobe vfio-pci
if [ "$?" -ne "0" ]; then echo -e "${colors[red]} Could not modprobe vfio-pci. Please fix this.${colors[none]}"; exit 1 ; fi
fi

lspciOutput="$(lspci -nn)" ; PCIDevices="$(grep -E "$pciREGEX" <<<"$lspciOutput")"
if [ -z "$PCIDevices" ]; then echo "Couldn't find any PCI devices with regex '$pciREGEX', please try another one. Panicking."; exit 1; fi
inc=0
while read PCIDevice
do
shortBuspath="$(cut -d' ' -f1<<<$PCIDevice)"
buspath="0000:$shortBuspath"
iommuGroup="$(basename $(readlink /sys/bus/pci/devices/$buspath/iommu_group))"
fullBuspath="0000:$shortBuspath"
iommuGroup="$(basename $(readlink /sys/bus/pci/devices/$fullBuspath/iommu_group))"

echo -e " Matched:\t${colors[blue]}$PCIDevice${colors[none]}"
echo -e " IOMMU Group:\t${colors[blue]}${iommuGroup}${colors[none]}"
Expand All @@ -295,52 +317,88 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O
vendor="$( cut -d':' -f1 <<<"$vendorClass")"
class="$( cut -d':' -f2 <<<"$vendorClass")"

if ! [[ $@ == *"restorebind"* ]]
if ! [[ $@ == *"restorebind"* ]] # Do our initial checks unless we're restoring an original driver binding right now
then
currentDriver="$(lspci -nn -k -s "$buspath" | grep -oP '(?<=driver in use:\ ).*')"
currentDriver="$(lspci -nn -k -s "$fullBuspath" | grep -oP '(?<=driver in use:\ ).*')"
if ! [ -z "$currentDriver" ]
then
deviceDrivers["$vendorClass"]="$currentDriver" # Add it to the device driver array.
driver="${deviceDrivers["$vendorClass"]}"
echo -e " [INFO] ${colors[green]}Detected driver ${colors[none]}${driver}${colors[green]} is using this device. It will be re-bound on VM exit.${colors[none]}"
if [ "$currentDriver" == "vfio-pci" ]
then
echo -e " [INFO] ${colors[green]}Already bound to $currentDriver, leaving alone.${colors[none]}"
else
echo -e " [INFO] ${colors[green]}Detected driver ${colors[none]}${driver}${colors[green]} is using this device. It will be re-bound on VM exit.${colors[none]}"
fi
else
echo -e " [INFO] ${colors[green]}Has no driver, it will be bound to vfio-pci and left that way for the remainder of this boot.${colors[none]}"
fi
fi
driver="${deviceDrivers["$vendorClass"]}"

driver="${deviceDrivers["$vendorClass"]}"

if [[ $@ == *"unbind"* ]] && ! isDry
then
if ! isDry && grep -o '\[VGA controller\]' <<<$(lspci -vnnn -s "$buspath") ; then gpuDetectedAndInUse "$busPath[$vendorClass]" ; fi
echo -e "${colors[green]} Unbinding from:\t${colors[none]}$driver" ; echo "$buspath" | sudo tee /sys/bus/pci/devices/$buspath/driver/unbind >/dev/null
if ! isDry && gpuCheck $fullBuspath
then
if [ ! -z "$driver" ] && [ ! "$driver" == "vfio-pci" ] ; then gpuDetectedAndInUse "$busPath[$vendorClass]" ; fi
fi
if [ ! -z "$driver" ] && [ "$driver" != "vfio-pci" ]
then
echo -e "${colors[green]} Unbinding from:\t${colors[none]}$driver"
echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/devices/$fullBuspath/driver/unbind >/dev/null # Try an unbind but give up after 5 seconds with a kill-9
if [ $? -eq 137 ]; then bindingTimeout unbind ; exit 1 ; fi

fi
elif [[ $@ == *"unbind"* ]] && isDry
then
echo -e " [DRY] ${colors[green]}Not unbinding it.${colors[none]}"
fi

if [[ $@ == *"vfiobind"* ]] && ! isDry
then
echo -e "${colors[green]} Binding to:\t\t${colors[none]}vfio-pci" ; echo "0x$vendor 0x$class" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id >/dev/null
if [ $? -ne 0 ]; then echo -e " ${colors[red]}Rebind to vfio-pci failed!${colors[none]}"; fi
sleep 1
echo 1 | sudo tee /sys/bus/pci/rescan >/dev/null
if ! [ "$currentDriver" == "vfio-pci" ]
then
echo -e "${colors[green]} Adding ID and binding to:\t${colors[none]}vfio-pci"
echo "0x$vendor 0x$class" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/new_id >/dev/null 2>&1
if [ $? -eq 137 ]; then bindingTimeout "bind via new_id" ; exit 1 ; fi
echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/bind >/dev/null 2>&1 # Try and bind the PCI address for this device just in case ID already added and not automatically bound
if [ $? -eq 137 ]; then bindingTimeout bind ; exit 1 ; fi
fi
while ! [ -c /dev/vfio/$iommuGroup ]
do
sleep 1 && iommuTimeout=$(((iommuTimeout+1)))
if [ $iommuTimeout -gt 5 ]; then echo -e "${colors[red]} Timed out waiting for bound device to appear under ${colors[none]}/dev/vfio${colors[none]}" ; exit 1 ; fi
done
permissionsManager takeown /dev/vfio/${iommuGroup}
echo 1 | sudo tee /sys/bus/pci/rescan >/dev/null
elif [[ $@ == *"vfiobind"* ]] && isDry
then
echo -e " [DRY] ${colors[green]}Not binding to:\t${colors[none]}vfio-pci"
echo -e " [DRY] ${colors[green]}Not binding the device.${colors[none]}"
echo
elif [[ $@ == *"restorebind"* ]] && ! isDry
then
if ! [ -z "$driver" ]
then
echo -e "${colors[green]} Rebinding $vendorClass back to driver:\t${colors[none]}$driver"
echo "0x$vendor 0x$class" | sudo tee /sys/bus/pci/drivers/vfio-pci/remove_id >/dev/null
echo "$buspath" | sudo tee /sys/bus/pci/devices/$buspath/driver/unbind >/dev/null
echo "$buspath" | sudo tee /sys/bus/pci/drivers/$driver/bind >/dev/null
echo "$fullBuspath" | sudo tee /sys/bus/pci/devices/$fullBuspath/driver/unbind > /dev/null
echo "$fullBuspath" | sudo tee /sys/bus/pci/drivers/$driver/bind > /dev/null
if [ "$?" -eq "0" ]; then echo -e "${colors[green]} Successfully rebound.${colors[none]}" ; else echo -e "${colors[red]} Was unable to rebind it to $driver."; fi
else
echo -e "${colors[green]} Device $vendorClass had no driver so left bound to vfio-pci.${colors[none]}"
fi
fi


pciArgs="$pciArgs -device vfio-pci,host=$buspath,id=hostdev$inc"
if gpuCheck $fullBuspath
then
echo -e "${colors[green]} This appears to be a GPU.\t${colors[none]}"
#pciArgs="$pciArgs -device vfio-pci,x-vga=on,multifunction=on,host=$fullBuspath,romfile="/path/to/romfile/example.rom",id=hostdev$inc" # Not yet implemented but commented for anyone willing to try a romfile
pciArgs="$pciArgs -device vfio-pci,host=$fullBuspath,id=hostdev$inc"
else
pciArgs="$pciArgs -device vfio-pci,host=$fullBuspath,id=hostdev$inc"
fi
inc=$((($inc+1)))
done <<<"$PCIDevices"

Expand Down Expand Up @@ -417,6 +475,9 @@ do
-imageformat|-format)
if [ ! -z "$latestDisk" ]; then disks[$latestDisk]="$2" ; fi # Add the format for the most recently declared disk
shift
;;
-looking-glass|-lookingglass|-lg)
lookingglass="1"
;;
-extras)
extras="$2"
Expand Down Expand Up @@ -564,13 +625,13 @@ if [[ ! -z "$bridgeArgs" ]]; then bridger start $bridgeArgs ; fi
cpuArgs="-cpu host,kvm=off"
if [ ! -z "$hyperv" ]
then
echo -e "${colors[green]}-hyperv\t\tspecified, HyperV Enlightenments will be enabled for this run.{$colors[none]}"
echo -e "${colors[green]}-hyperv\t\tspecified, HyperV Enlightenments will be enabled for this run.${colors[none]}"
cpuArgs="${cpuArgs},hv-frequencies,hv-relaxed,hv-reset,hv-runtime,hv-spinlocks=0x1fff,hv-stimer,hv-synic,hv-time,hv-vapic,hv-vpindex"
fi

smpArgs="-smp sockets=1,cores=$guestCores,threads=$guestThreads"
coreArgs="$machineArgs -enable-kvm -m $guestmemoryMB $cpuArgs $smpArgs"
coreArgs="$coreArgs -drive if=pflash,format=raw,unit=0,readonly,file=$biosPath"
coreArgs="$coreArgs -drive if=pflash,format=raw,unit=0,readonly=on,file=$biosPath"

if [[ ! -z "${!disks[@]}" ]]
then
Expand Down Expand Up @@ -601,6 +662,22 @@ fi

if [[ ! -z "$usbREGEX" ]] && [[ ! -z "$pciREGEX" ]] || [[ -z "$DISPLAY" ]] ; then coreArgs="$coreArgs -nographic -vga none" ; fi # Create X11 window if no passthrough used. Also if $DISPLAY isn't set.

if [ ! -z "$lookingglass" ]
then
echo -e "${colors[green]}-lookingglass\tspecified, we'll start qemu with a spice server and shared memory for it.${colors[none]}"
# Bad way to do this but wanted to keep them readable individually
lgArgs="$lgArgs -object {'qom-type':'memory-backend-file','id':'shmmem-shmem0','mem-path':'/dev/shm/looking-glass','size':67108864,'share':true}" # Memory currently hardcoded to 64M, can be changed in this script if needed. Plan to take resolution as an argument in future.
lgArgs="$lgArgs -device ivshmem-plain,id=shmem0,memdev=shmmem-shmem0,id=lookingglass"
lgArgs="$lgArgs -chardev spicevmc,id=charchannel0,name=vdagent"
lgArgs="$lgArgs -device virtio-serial"
lgArgs="$lgArgs -chardev socket,path=/dev/shm/vmsocket,server=on,wait=off,id=vmsocket"
lgArgs="$lgArgs -device virtserialport,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0"
lgArgs="$lgArgs -audiodev id=audio1,driver=spice"
lgArgs="$lgArgs -spice port=5900,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on"
lgArgs="$lgArgs -chardev spicevmc,id=charredir0,name=usbredir"
lgArgs="$lgArgs -chardev spicevmc,id=charredir1,name=usbredir"
fi

if [ ! -z "$extras" ]
then
echo -e "${colors[green]}-extras\t\tspecified, we'll start qemu with these extra arguments:${colors[none]}\t$extras"
Expand Down Expand Up @@ -641,17 +718,18 @@ then
echo "---------------------"
echo "Here are the completed args from this DRY run:"
echo -e "Core:\n${colors[blue]} $coreArgs $hugeArgs${colors[none]}"
if [ ! -z "$networkArgs" ]; then echo -e "Net :\n${colors[blue]}${networkArgs}${colors[none]}" ; fi
if [ ! -z "$usbArgs" ] ; then echo -e "USB :\n${colors[blue]}${usbArgs}${colors[none]}" ; fi
if [ ! -z "$pciArgs" ] ; then echo -e "PCI :\n${colors[blue]}${pciArgs}${colors[none]}" ; fi
if [ ! -z "$extras" ] ; then echo -e "Extras:\n${colors[blue]}${extras}${colors[none]}" ; fi
if [ ! -z "$networkArgs" ]; then echo -e "Net :\n${colors[blue]}${networkArgs}${colors[none]}" ; fi
if [ ! -z "$usbArgs" ] ; then echo -e "USB :\n${colors[blue]}${usbArgs}${colors[none]}" ; fi
if [ ! -z "$pciArgs" ] ; then echo -e "PCI :\n${colors[blue]}${pciArgs}${colors[none]}" ; fi
if [ ! -z "$lgArgs" ] ; then echo -e "Looking Glass:\n${colors[blue]}${lgArgs}${colors[none]}" ; fi
if [ ! -z "$extras" ] ; then echo -e "Extras:\n${colors[blue]}${extras}${colors[none]}" ; fi
echo -ne "\nRun the script with the same arguments again and include ${colors[red]}-run${colors[none]} to actually pass these to qemu-system-x86_64\n"
exit 0
else
sudo prlimit --pid $$ --memlock=unlimited:unlimited
set -m # Allow scripted bash job controls (fg, bg, etc)
echo -ne "Starting qemu now.\n\n"
qemu-system-x86_64 $coreArgs $hugeArgs $networkArgs $usbArgs $pciArgs $extras &
qemu-system-x86_64 $coreArgs $hugeArgs $networkArgs $usbArgs $pciArgs $lgArgs $extras &
sudo chrt -p 1 -p $! >/dev/null # Set qemu to have high priority
sudo taskset -cp ${tasksetThreads} $! >/dev/null # Set the permitted host threads
fg
Expand Down

0 comments on commit e649cb6

Please sign in to comment.