diff --git a/README.md b/README.md index 22c2e46..e85b576 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,11 @@ Using example 2: Set how much memory the VM gets for this run. Argument assumes megabytes unless you explicitly use a suffix like K, M, or G. If this argument isn't specified the default value is HALF of the host total. -`-hugepages / -huge` +`-hugepages / -huge / -hugepages /optional/mountpoint/to/custom/hugepage` - Try to mount (if not already) and allocate some hugepages based on the VM's total memory defined with -memory (or default). - If successful, qemu is given the arguments to use it. Gets unallocated after VM exit in the cleanup routine. + Tries to allocate hugepages for the VM based on it's total memory defined with -memory. Hugepages for a VM with frequent random memory access such as gaming contexts can be much snappier than your regular process doing the same thing across many small pages. + If no argument is given it'll use /dev/hugepages which is often 2MB per page. The script drops host memory cache and then compact_memory before allocating to aim for less hugepage fragmentation and will clean up after itself by setting pages back to 0 to reclaim memory for the host to use later. + Also supports preallocation, so if you reserved some 1GB pages at boot time and mounted a pagesize=1G hugetlbfs to some directory, specifying that can skip the above line if there's enough for the VM to use. Allocating hugepages at boot time sacrifices a lot of memory but doing it so early prevents fragmentation. `-hyperv` @@ -158,6 +159,11 @@ This example would catch any: If set adds extra arbitrary commands to the cmdline of qemu (Once invoked) Useful for `-device ac97` to get some quick and easy audio support if the host is running pulseaudio. + +`-romfile/-vbios /path/to/vbios.bin` + Accepts a file path to a rom file to use on any detected GPU during -pci argument processing. + You should check the vbios dump you're about to use is safe before using it on a GPU, rom-parser is a good project for this. + Otherwise you can often download your model's vbios from TechPowerup and patch it with a project like NVIDIA-vBIOS-VFIO-Patcher before using it. `-run` diff --git a/main b/main index 44af86c..18836dd 100755 --- a/main +++ b/main @@ -41,64 +41,70 @@ fi # function printHelp { - echo -e "This script exists to make various vfio activity less convoluted. Among other use-cases" - echo -e "such as LiveCD usage, testing if drives boot, PXE imaging, or any thing else with optional PCI passthrough." - echo -e "Hopefully somebody finds it useful.\n" + printer "This script exists to make various vfio activity less convoluted. Among other use-cases" + printer "such as LiveCD usage, testing if drives boot, PXE imaging, or any thing else with optional PCI passthrough." + printer "Hopefully somebody finds it useful.\n" - echo -e "Valid example arguments: [${colors[blue]}There's more in the README.md!${colors[none]}]\n" + printer "Valid example arguments: [ ${colors[blue]}There's more in the README.md! ${colors[none]}]\n" - echo -e "[Required for most use-cases]" - echo -e " ${colors[cyan]}-image /dev/zvol/zpool/windows${colors[none]}" - echo -e " ${colors[cyan]}-imageformat${colors[none]}/${colors[cyan]}-format${colors[none]} raw/qcow/vdi/etc${colors[none]}" - echo -e " A file or blockdevice to give the guest. Complete with it's [Qemu recognised] format\n" + printer " ${colors[cyan]}-image /dev/zvol/zpool/windows" + printer " ${colors[cyan]}-imageformat/${colors[cyan]}-format [raw / qcow / vdi / etc]" + printer " A file or blockdevice to give the guest. Complete with it's [Qemu recognised] format\n" - echo -e "\n[For first-install or liveCD booting]" - echo -e " ${colors[cyan]}-iso /path/to/a/diskimage.iso${colors[none]}" - echo -e " If you're installing the OS for drivers you may wish to include this\n" + printer " ${colors[cyan]}-iso /path/to/a/diskimage.iso" + printer " If you're installing the OS for drivers you may wish to include this\n" - echo -e "\n[Extra useful flags and for vm runtime]" - echo -e " ${colors[cyan]}-iommugroups / -iommugrouping${colors[none]}" - echo -e " (Try to) Print IOMMU Groups then exit. Useful for the -PCI arg.\n" + printer " ${colors[cyan]}-iommugroups / -iommugrouping" + printer " (Try to) Print IOMMU Groups then exit. Useful for the -PCI arg.\n" - echo -e " ${colors[cyan]}-bridge br0,tap0${colors[none]}\t(Attach vm's tap0 to existing br0)" - echo -e " ${colors[cyan]}-bridge br0,tap0,eth0${colors[none]}\t(Create br0, attach vm's tap0 and host's eth0 to br0, use dhclient for a host IP.)" - echo -e " If no network options are specified the default qemu NAT adapter will be used.\n" + printer " ${colors[cyan]}-bridge br0,tap0\t${colors[none]}(Attach vm's tap0 to existing br0)" + printer " ${colors[cyan]}-bridge br0,tap0,eth0\t${colors[none]}(Create br0, attach vm's tap0 and host's eth0 to br0, use dhclient for a host IP.)" + printer " If no network options are specified the default qemu NAT adapter will be used.\n" - echo -e " ${colors[cyan]}-nonet${colors[none]}" - echo -e " If specified, the VM is not given a network adapter. Useful if you're passing a USB or PCI network card.\n" + printer " ${colors[cyan]}-nonet" + printer " If specified, the VM is not given a network adapter. Useful if you're passing a USB or PCI network card.\n" - echo -e " ${colors[cyan]}-memory 8192M${colors[none]} / ${colors[cyan]}-memory 8G${colors[none]} / ${colors[cyan]}-mem xxxxM${colors[none]}" - echo -e " Set VM memory (Default is half of system total memory)\n" + printer " ${colors[cyan]}-memory 8192M / ${colors[cyan]}-memory 8G / ${colors[cyan]}-mem xxxxM" + printer " Set VM memory (Default is half of system total memory)\n" - echo -e " ${colors[cyan]}-hugepages${colors[none]} / ${colors[cyan]}-huge${colors[none]}" - echo -e " Try to mount & allocate hugepages for qemu with the -mem specified." - echo -e " (May want to specify -mem for this. Half of host ram may be too much.)\n" + printer " ${colors[cyan]}-hugepages / ${colors[cyan]}-huge /optional/hugepage/mount" + printer " Try to allocate hugepages for qemu with the -mem value specified." + printer " If no hugepage mountpoint given as argument, tries /dev/hugepages which is often present." + printer " Supports hugepage preallocation from boot time and will use existing pages if free." + printer " (May want to specify -mem for this. Half of host ram may be too much.)\n" - echo -e " ${colors[cyan]}-hyperv${colors[none]}" - echo -e " Enable hyper-v enlightenments for nested virtualization." - echo -e " Can help invasive anticheats play along however HyperV will need to be enabled in the guest's Features.\n" + printer " ${colors[cyan]}-hyperv" + printer " Enable hyper-v enlightenments for nested virtualization." + printer " Can help invasive anticheats play along however HyperV will need to be enabled in the guest's Features.\n" - echo -e " ${colors[cyan]}-usb 'SteelSeries|Keyboard|Xbox|1234:5678'${colors[none]}" - echo -e " A regex for lsusb parsing. Can be anything that matches a specific device line.\n" + printer " ${colors[cyan]}-usb 'SteelSeries|Keyboard|Xbox|1234:5678'" + printer " A regex for lsusb parsing. Matches against device lines from lsusb output.\n" - echo -e " ${colors[cyan]}-pci 'Realtek|NVIDIA|10ec:8168'${colors[none]}" - echo -e " A regex for lspci parsing. Example will take any nvidia and Realtek cards, plus any device with the ID.\n" + printer " ${colors[cyan]}-pci 'Realtek|NVIDIA|10ec:8168'" + printer " A regex for lspci parsing. Matches against device lines from lspci output.\n" - echo -e " ${colors[cyan]}-looking-glass ${colors[none]}/${colors[cyan]} -lookingglass ${colors[none]}/${colors[cyan]} -lg ${colors[none]}" - echo -e " Adds a 64MB shared memory module for the Looking Glass project and a spice server onto the guest for input from host during Looking Glass usage.\n" + printer " ${colors[cyan]}-looking-glass / -lookingglass / -lg " + printer " Adds a 64MB shared memory module for the Looking Glass project and a spice server onto the guest for input from host during Looking Glass usage.\n" - echo -e " ${colors[cyan]}-extras '-device abc123,b,c,d=e' / -extras '-device AC97' ${colors[none]}" - echo -e " Accepts a string for arbitrary extra qemu arguments. Highly useful when I want to use extra qemu features without having to implement them into the script right away.\n" + printer " ${colors[cyan]}-extras '-device abc123,b,c,d=e' / -extras '-device AC97' " + printer " Accepts a string for arbitrary extra qemu arguments. Highly useful when I want to use extra qemu features without having to implement them into the script right away.\n" - echo -e " ${colors[cyan]}-taskset 0,1,2,3,4,5,6 / ${colors[cyan]}-taskset 0,2,4,8${colors[none]}" - echo -e " A comma delimited list of host threads the Guest is allowed to execute on." - echo -e " (Setting this also shapes the guest CPU topology to match.)" - echo -e " If you've configured core isolation on the host, this is the argument for you!\n" + printer " ${colors[cyan]}-romfile/-vbios path-to-vbios.bin" + printer " Accepts a file path to a rom file to use on any detected GPU during -pci argument processing. You should check the vbios dump you're about to use is safe before using it on a GPU, rom-parser is a good project for this.\n" - echo -e "Please see the README.md file for more information about the flags." + printer " ${colors[cyan]}-taskset 0,1,2,3,4,5,6${colors[cyan]} / ${colors[cyan]}-taskset 0,2,4,8" + printer " A comma delimited list of host threads the Guest is allowed to execute on." + printer " (Setting this also shapes the guest CPU topology to match.)" + printer " If you've configured core isolation on the host, this is the argument for you!\n" + + printer "Please see the README.md file for more information about the flags." exit 1 } +function printer { + echo -e "${@} ${colors[none]}" +} + function permissionsManager { # chown's things before qemu starts, but also tries to return them in the end. if isDry ; then return ; fi # Do nothing when dry. @@ -132,13 +138,15 @@ function permissionsManager { # chown's things before qemu starts, but also trie function isDry { if [ $DRY -eq 1 ] ; then return 0 ; else return 1 ; fi ; } # Read numeric value from meminfo -function readMeminfo { if [ ! -z "$1" ]; then grep -m1 "$1" /proc/meminfo | grep -Eo '[0-9]+'; fi ; } +function readMeminfo { if [ ! -z "$1" ]; then grep -m1 "$1" /proc/meminfo | grep -Eo '[0-9]+'; fi ; } +function readHugeInfo { if [ ! -z "$1" ] && [ ! -z "$2" ]; then cat /sys/kernel/mm/hugepages/hugepages-${1}kB/${2}; fi ; } +function writeHugeInfo { if [ ! -z "$1" ] && [ ! -z "$2" ]; then sudo tee /sys/kernel/mm/hugepages/hugepages-${1}kB/${2} > /dev/null; fi ; } function gpuCheck { if grep -o '\[VGA controller\]' <<<$(lspci -vnnn -s "$1") > /dev/null 2>&1 ; then return 0 ; else return 1 ; fi } function bindingTimeout { if [ ! -z "$1" ]; then attemptedTask=$1 ; else attemptedTask="bind or unbind" ; fi - echo -e "${colors[red]} The device ${colors[none]} $fullBuspath ${colors[red]}// ${colors[none]}$vendorClass ${colors[red]} Was unable to $attemptedTask after 5 seconds, is something else using it? (E.g This will happen to a GPU in use by X). Giving up.${colors[none]}" + printer "${colors[red]} The device $fullBuspath ${colors[red]}// $vendorClass ${colors[red]} Was unable to $attemptedTask after 5 seconds, is something else using it? (E.g This will happen to a GPU in use by X). Giving up." exit 1 } @@ -146,7 +154,7 @@ function bindingTimeout { function do_cleanup { echo "Cleaning up.." permissionsManager return # Restore original permissions if any - echo -ne "${colors[none]}" + echo -ne "" if ! isDry then # Undo bridge if used @@ -161,14 +169,13 @@ function do_cleanup { if [ "$dm" == "seen" ]; then echo "Attempting to restore display-manager..." ; sudo systemctl start display-manager ; fi # Start DM if seen before run. fi - if [ "$HUGEPAGES" == "1" ] # Clean up hugepages + if [ ! -z "$HUGEPAGES" ] && [ ! -z "$hugepagesSelfAllocated" ] # Clean up hugepages then - echo 0 | sudo tee /proc/sys/vm/nr_hugepages >/dev/null # pack up - if [ "$hugeMount" == "self" ]; then umount $hugePath ; fi #umount if we mounted it + echo 0 | writeHugeInfo ${hugepageSizeKB} nr_hugepages # pack up fi fi [ -f '/dev/shm/looking-glass' ] && rm /dev/shm/looking-glass - echo -e "\n${colors[green]}Cleanup complete.${colors[none]}" + printer "\n${colors[green]}Cleanup complete." exit } @@ -221,24 +228,24 @@ function bridger { permissionsManager takeown /dev/net/tun* echo '------------------' echo -ne "Bridge details:\n\n" ; brctl show $br ; echo - if [ $bridgePreexists -eq 0 ]; then echo "Running dhclient on $br..." ; sudo dhclient -v $br 2>&1|grep '^DHCPACK' ; else echo -e "${colors[green]}Bridge already existed, not running dhclient -r on it.${colors[none]}" ; fi + if [ $bridgePreexists -eq 0 ]; then echo "Running dhclient on $br..." ; sudo dhclient -v $br 2>&1|grep '^DHCPACK' ; else printer "${colors[green]}Bridge already existed, not running dhclient -r on it." ; fi echo '------------------' elif [ "$mode" == "start" ] && isDry then - echo -e "${colors[green]}-bridge\t\tacknowledged [DRY]${colors[none]}" + printer "${colors[green]}-bridge\t\tacknowledged [DRY]" elif [ "$mode" == "stop" ] && ! isDry then if [ $bridgePreexists -eq 0 ] then - echo -e "${colors[yellow]}Undoing our bridge..." - dhclient -r $br >/dev/null 2>&1 && echo -e "${colors[green]}dhcp lease released and dhclient process ended...${colors[none]}" + printer "${colors[yellow]}Undoing our bridge..." + dhclient -r $br >/dev/null 2>&1 && printer "${colors[green]}dhcp lease released and dhclient process ended..." for i in $tap $br; do sudo ip link set $i down ; sudo ip link del $i; done # Remove the tap and bridge - if ! ip link show br0 >/dev/null 2>&1; then echo -e "${colors[green]}Bridge removed.${colors[none]}"; fi + if ! ip link show br0 >/dev/null 2>&1; then printer "${colors[green]}Bridge removed."; fi else - echo -e "${colors[green]}We only attached $tap to an existing bridge this run, removing $tap.${colors[none]}" - for i in $tap ; do sudo ip link set $i down ; sudo ip link del $i; done && echo -e "${colors[green]}$tap removed.${colors[none]}" # Remove the tap + printer "${colors[green]}We only attached $tap to an existing bridge this run, removing $tap." + for i in $tap ; do sudo ip link set $i down ; sudo ip link del $i; done && printer "${colors[green]}$tap removed." # Remove the tap fi - if [ "$nm" == "seen" ]; then sudo systemctl restart NetworkManager ; echo -e "${colors[green]}NetworkManager restarted.${colors[none]}" ; fi + if [ "$nm" == "seen" ]; then sudo systemctl restart NetworkManager ; printer "${colors[green]}NetworkManager restarted." ; fi fi } @@ -268,26 +275,26 @@ function enumerateUSBs { # This only makes arguments then newUsbArg="-device usb-host,vendorid=0x$vendor,productid=0x$product" usbArgs="$usbArgs $newUsbArg" - echo -e " Matched: ${colors[blue]}$vendor:$product '$Name'${colors[none]}" - echo -e "${colors[green]} Added to USB Args as:\t$newUsbArg\n${colors[none]}" + printer " Matched: ${colors[blue]}$vendor:$product '$Name'" + printer "${colors[green]} Added to USB Args as:\t$newUsbArg\n" else - echo -e "${colors[red]}Skipping: '$USBDevice' as there was an issue finding it.\n" + printer "${colors[red]}Skipping: '$USBDevice' as there was an issue finding it.\n" fi done <<<"$USBDevices" } function enumeratePCIs { # This makes arguments and also unbinds from drivers. Optional vfio-pci rebinding. function gpuDetectedAndInUse { - echo -e "${colors[yellow]} Video device $1 is bound to a driver which isn't vfio-pci and could be in use by the DM, framebuffer or otherwise.${colors[none]}" - echo -e "${colors[yellow]} For this reason the script will now attempt to stop the display-manager service and unbind " - echo -e "${colors[yellow]} the efi framebuffer instead of risking a driver unbind deadlock in waiting for X to quit.${colors[none]}" - echo -e "${colors[yellow]} If your X server and virtual consoles don't use this card you can unbind it from its driver manually before running this script.${colors[none]}" + printer "${colors[yellow]} Video device $1 is bound to a driver which isn't vfio-pci and could be in use by the DM, framebuffer or otherwise." + printer "${colors[yellow]} For this reason the script will now attempt to stop the display-manager service and unbind " + printer "${colors[yellow]} the efi framebuffer instead of risking a driver unbind deadlock in waiting for X to quit." + printer "${colors[yellow]} If your X server and virtual consoles don't use this card you can unbind it from its driver manually before running this script." if systemctl is-active display-manager >/dev/null && ! isDry then - echo -e "${colors[yellow]} Stopping display-manager and unbinding console drivers in 5 seconds...${colors[none]}" ; sleep 5 + printer "${colors[yellow]} Stopping display-manager and unbinding console drivers in 5 seconds..." ; sleep 5 sudo systemctl stop display-manager ; export dm="seen" else - echo -e "${colors[yellow]} Could not find a service for display-manager. Confused but trying to continue anyway.${colors[none]}" + printer "${colors[yellow]} Could not find a service for display-manager. Confused but trying to continue anyway." fi consoleDrivers unbind } @@ -296,9 +303,9 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O declare -A -g deviceDrivers # Array for tracking devices and their driver. if ! lsmod|grep -q vfio_pci; then - echo -e "${colors[yellow]} vfio-pci isn't loaded. Loading it now.${colors[none]}" + printer "${colors[yellow]} vfio-pci isn't loaded. Loading it now." sudo modprobe vfio-pci - if [ "$?" -ne "0" ]; then echo -e "${colors[red]} Could not modprobe vfio-pci. Please fix this.${colors[none]}"; exit 1 ; fi + if [ "$?" -ne "0" ]; then printer "${colors[red]} Could not modprobe vfio-pci. Please fix this."; exit 1 ; fi fi lspciOutput="$(lspci -nn)" ; PCIDevices="$(grep -E "$pciREGEX" <<<"$lspciOutput")" @@ -310,8 +317,8 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O fullBuspath="0000:$shortBuspath" iommuGroup="$(basename $(readlink /sys/bus/pci/devices/$fullBuspath/iommu_group))" - echo -e " Matched:\t${colors[blue]}$PCIDevice${colors[none]}" - echo -e " IOMMU Group:\t${colors[blue]}${iommuGroup}${colors[none]}" + printer " Matched:\t${colors[blue]}$PCIDevice" + printer " IOMMU Group:\t${colors[blue]}${iommuGroup}" vendorClass="$( grep -Eo '(([0-9]|[a-f]){4}|:){3}' <<<$PCIDevice )" vendor="$( cut -d':' -f1 <<<"$vendorClass")" @@ -326,12 +333,12 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O driver="${deviceDrivers["$vendorClass"]}" if [ "$currentDriver" == "vfio-pci" ] then - echo -e " [INFO] ${colors[green]}Already bound to $currentDriver, leaving alone.${colors[none]}" + printer " [INFO] ${colors[green]}Already bound to $currentDriver, leaving alone." else - echo -e " [INFO] ${colors[green]}Detected driver ${colors[none]}${driver}${colors[green]} is using this device. It will be re-bound on VM exit.${colors[none]}" + printer " [INFO] ${colors[green]}Detected driver ${driver}${colors[green]} is using this device. It will be re-bound on VM exit." fi else - echo -e " [INFO] ${colors[green]}Has no driver, it will be bound to vfio-pci and left that way for the remainder of this boot.${colors[none]}" + printer " [INFO] ${colors[green]}Has no driver, it will be bound to vfio-pci and left that way for the remainder of this boot." fi fi @@ -345,21 +352,21 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O fi if [ ! -z "$driver" ] && [ "$driver" != "vfio-pci" ] then - echo -e "${colors[green]} Unbinding from:\t${colors[none]}$driver" + printer "${colors[green]} Unbinding from:\t${colors[none]}$driver" echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/devices/$fullBuspath/driver/unbind >/dev/null # Try an unbind but give up after 5 seconds with a kill-9 if [ $? -eq 137 ]; then bindingTimeout unbind ; exit 1 ; fi fi elif [[ $@ == *"unbind"* ]] && isDry then - echo -e " [DRY] ${colors[green]}Not unbinding it.${colors[none]}" + printer " [DRY] ${colors[green]}Not unbinding it." fi if [[ $@ == *"vfiobind"* ]] && ! isDry then if ! [ "$currentDriver" == "vfio-pci" ] then - echo -e "${colors[green]} Adding ID and binding to:\t${colors[none]}vfio-pci" + printer "${colors[green]} Adding ID and binding to:\t${colors[none]}vfio-pci" echo "0x$vendor 0x$class" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/new_id >/dev/null 2>&1 if [ $? -eq 137 ]; then bindingTimeout "bind via new_id" ; exit 1 ; fi echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/bind >/dev/null 2>&1 # Try and bind the PCI address for this device just in case ID already added and not automatically bound @@ -368,34 +375,37 @@ function enumeratePCIs { # This makes arguments and also unbinds from drivers. O while ! [ -c /dev/vfio/$iommuGroup ] do sleep 1 && iommuTimeout=$(((iommuTimeout+1))) - if [ $iommuTimeout -gt 5 ]; then echo -e "${colors[red]} Timed out waiting for bound device to appear under ${colors[none]}/dev/vfio${colors[none]}" ; exit 1 ; fi + if [ $iommuTimeout -gt 5 ]; then printer "${colors[red]} Timed out waiting for bound device to appear under /dev/vfio" ; exit 1 ; fi done permissionsManager takeown /dev/vfio/${iommuGroup} echo 1 | sudo tee /sys/bus/pci/rescan >/dev/null elif [[ $@ == *"vfiobind"* ]] && isDry then - echo -e " [DRY] ${colors[green]}Not binding the device.${colors[none]}" + printer " [DRY] ${colors[green]}Not binding the device." echo elif [[ $@ == *"restorebind"* ]] && ! isDry then if ! [ -z "$driver" ] then - echo -e "${colors[green]} Rebinding $vendorClass back to driver:\t${colors[none]}$driver" + printer "${colors[green]} Rebinding $vendorClass back to driver:\t${colors[none]}$driver" echo "0x$vendor 0x$class" | sudo tee /sys/bus/pci/drivers/vfio-pci/remove_id >/dev/null echo "$fullBuspath" | sudo tee /sys/bus/pci/devices/$fullBuspath/driver/unbind > /dev/null echo "$fullBuspath" | sudo tee /sys/bus/pci/drivers/$driver/bind > /dev/null - if [ "$?" -eq "0" ]; then echo -e "${colors[green]} Successfully rebound.${colors[none]}" ; else echo -e "${colors[red]} Was unable to rebind it to $driver."; fi + if [ "$?" -eq "0" ]; then printer "${colors[green]} Successfully rebound." ; else printer "${colors[red]} Was unable to rebind it to $driver."; fi else - echo -e "${colors[green]} Device $vendorClass had no driver so left bound to vfio-pci.${colors[none]}" + printer "${colors[green]} Device $vendorClass had no driver so left bound to vfio-pci." fi fi if gpuCheck $fullBuspath then - echo -e "${colors[green]} This appears to be a GPU.\t${colors[none]}" - #pciArgs="$pciArgs -device vfio-pci,x-vga=on,multifunction=on,host=$fullBuspath,romfile="/path/to/romfile/example.rom",id=hostdev$inc" # Not yet implemented but commented for anyone willing to try a romfile - pciArgs="$pciArgs -device vfio-pci,host=$fullBuspath,id=hostdev$inc" + printer "${colors[green]} This appears to be a GPU.\t" + if [ ! -z "$romfilePath" ] && [ -f "$romfilePath" ] + then + printer "${colors[green]} Romfile will be passed to the GPU on VM start.\t" + pciArgs="$pciArgs -device vfio-pci,x-vga=on,multifunction=on,host=$fullBuspath,romfile=$romfilePath,id=hostdev$inc" + fi else pciArgs="$pciArgs -device vfio-pci,host=$fullBuspath,id=hostdev$inc" fi @@ -418,9 +428,9 @@ while [ $# -gt 0 ] do case "$(tr '[:upper:]' '[:lower:]'<<<$1)" in -colortest|-colourtest) - echo -e "${colors[green]}Ok, printing a color test then exiting...${colors[none]}" - for i in $(sort <<< ${!colors[@]}) ; do echo -e "Test for: ${colors[$i]}$i ${colors[none]}" ; done - echo -ne "${colors[none]}" + printer "${colors[green]}Ok, printing a color test then exiting..." + for i in $(sort <<< ${!colors[@]}) ; do printer "Test for: ${colors[$i]}$i " ; done + echo -ne "" exit $? ;; -nocolor|-nocolors|-nocolor|-nocolors) @@ -442,6 +452,7 @@ do ;; -huge|-hugepages) HUGEPAGES="1" + if [ -d "$2" ]; then hugePath="$2" ; shift ; fi ;; -usb) usbREGEX="$2" @@ -484,7 +495,11 @@ do shift ;; -hyperv) - hyperv='enabled' + hyperv="1" + ;; + -romfile|-vbios) + romfilePath="$2" + shift ;; --help|-help) printHelp @@ -505,15 +520,15 @@ done if [ "$printIOMMU" = "1" ] then - echo -e "${colors[green]}Ok, printing IOMMU Groups then exiting...${colors[none]}" + printer "${colors[green]}Ok, printing IOMMU Groups then exiting..." iommuDir="/sys/kernel/iommu_groups"; - if [ -d $iommuDir ]; then for g in `ls -1v $iommuDir`; do echo "IOMMU Group $g"; for d in $iommuDir/$g/devices/* ; do echo -e "${colors[cyan]}\t$(lspci -nns ${d##*/})${colors[none]}"; done; done ; else echo "Couldn't find $iommuDir" ; fi + if [ -d $iommuDir ]; then for g in `ls -1v $iommuDir`; do echo "IOMMU Group $g"; for d in $iommuDir/$g/devices/* ; do printer "${colors[cyan]}\t$(lspci -nns ${d##*/})"; done; done ; else echo "Couldn't find $iommuDir" ; fi exit $? fi if isDry then - echo -e "${colors[magenta]}This is a DRY run. Please specify ${colors[red]}-run${colors[magenta]} to actually run${colors[none]}" + printer "${colors[magenta]}This is a DRY run. Please specify ${colors[red]}-run${colors[magenta]} to actually run" else # Set a trap to run the cleanup function. trap do_cleanup EXIT HUP INT TERM @@ -530,11 +545,11 @@ hostThreads=$(grep -Pom1 'siblings.*\ \K([0-9]+)$' /proc/cpuinfo ) if [ -z "$tasksetThreads" ] then guestCores=$hostCores ; if [ ! -z $hostIsThreaded ]; then guestThreads=2; else guestThreads=1; fi - echo -e "${colors[yellow]}-taskset\tnot specified, guest gets full host CPU:\t${colors[none]}($hostCores) with ($hostThreads) threads." + printer "${colors[yellow]}-taskset\tnot specified, guest gets full host CPU:\t${colors[none]}($hostCores) with ($hostThreads) threads." else if grep -qE '^([0-9]|,)+$' <<< $tasksetThreads then - echo -e "${colors[green]}-taskset\tspecified, Guest will run on host CPU threads:\t${colors[none]}$tasksetThreads" + printer "${colors[green]}-taskset\tspecified, Guest will run on host CPU threads:\t$tasksetThreads" OLDIFS="$IFS" ; IFS=, ; tasksetCount=$(echo $tasksetThreads| wc -w) ; IFS="$OLDIFS" # Check if specified -taskset thread count is divisible by 2. Use hyperthreading/smt if supported if (( $tasksetCount % 2 == 0 )) && [ "$hostIsThreaded" == "1" ] ; then guestCores=$((( $tasksetCount / 2 ))) ; guestThreads=2; else guestCores=$tasksetCount ; guestThreads=1 ; fi @@ -550,9 +565,9 @@ then if [ ! -z "$totalHostMemoryKB" ] then guestmemoryMB="$((($totalHostMemoryKB / 1024 / 2)))" - echo -e "${colors[yellow]}-memory\t\tnot specified, will use half host total:\t${colors[none]}${guestmemoryMB} MB${colors[none]}" + printer "${colors[yellow]}-memory\t\tnot specified, will use half host total:\t${colors[none]}${guestmemoryMB} MB" else - echo "${colors[red]}Failed to find a default memory value for the guest.${colors[none]}" + echo "${colors[red]}Failed to find a default memory value for the guest." fi else if [[ "$memoryArgs" =~ 'G' ]]; then memoryArgs="$((($(grep -Eo '[0-9]+' <<< $memoryArgs) * 1024 )))"; fi # Convert to MB @@ -565,54 +580,85 @@ then defaultBiosPath="/usr/share/ovmf/x64/OVMF_CODE.fd" if [[ ! -f "$defaultBiosPath" ]] then - echo -e "${colors[red]}-bios\t\tnot specified and couldn't find default '$defaultBiosPath'. Please install OVMF or set your .fd file with the -bios argument" + printer "${colors[red]}-bios\t\tnot specified and couldn't find default '$defaultBiosPath'. Please install OVMF or set your .fd file with the -bios argument" else - echo -e "${colors[green]}-bios\t\tnot specified, using discovered default:\t${colors[none]}${defaultBiosPath}" + printer "${colors[green]}-bios\t\tnot specified, using discovered default:\t${colors[none]}${defaultBiosPath}" biosPath=$defaultBiosPath fi +else + if [ ! -f "$biosPath" ] ; then printer "${colors[red]}Bios path doesn't appear to exist: ${colors[none]}\t${biosPath}" ; exit 1 ; fi fi -# Eval Hugepages -hugePath="/dev/hugepages" -if [ "$HUGEPAGES" == "1" ] && ! isDry + # Eval Hugepages +if [ "$HUGEPAGES" == "1" ] then - Hugepagesize="$(readMeminfo Hugepagesize)" ; HugepagesizeMB="$((( $Hugepagesize / 1024 )))" - pagesRequired="$((($((($guestmemoryMB / $HugepagesizeMB))) + 1)))" - echo -ne "\nHugepages enabled for this run (Pagesize=${HugepagesizeMB}MB, $pagesRequired pages deployed. for the VM's ${guestmemoryMB}MB memory.)\n" - if grep -q "$hugePath" /proc/mounts + if [ -z "$hugePath" ] then - hugeMount=found && hugeArgs="-mem-path $hugePath" + hugePath="/dev/hugepages" ; hugePathDefault=1 + printer "${colors[yellow]}-hugepages\tspecified for this run but no mountpoint argument specified." + printer "${colors[yellow]}\t\tAttempting to use kernel default ${colors[none]}${hugePath}." else - echo "/dev/hugepages not mounted, attempting to mount it." - mount -t hugetlbfs hugetlbfs -o mode=01770,gid=0 $hugePath && hugeMount=self && hugeArgs="-mem-path $hugePath" # So we can umount later + printer "${colors[green]}-hugepages\tspecified with custom mountpoint." ; hugePathDefault=0 fi - echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null && echo -ne "Memory cache dropped for hugepages...\n" - echo -ne "running compact_memory... " ; echo 1 | sudo tee /proc/sys/vm/compact_memory >/dev/null && echo -ne "OK.\n" - echo "$pagesRequired" | sudo tee /proc/sys/vm/nr_hugepages >/dev/null ; echo -ne "Giving hugepages a moment to allocate." - while [ $(readMeminfo HugePages_Free) -lt $pagesRequired ] - do - echo -ne '.' - count=$((($count+1))) ; if [ $count -gt 30 ]; then break; fi - sleep 1 - done ; echo '' # Newline - - HugePages_Free="$(readMeminfo HugePages_Free)" - if [[ "$(readMeminfo HugePages_Free)" -lt "$pagesRequired" ]] + if ! grep "hugetlbfs ${hugePath}" /proc/mounts >/dev/null 2>&1 # Check for a hugetlbfs mount for this path then - echo -ne "${colors[red]}Error, couldn't allocate all required pages. Please free up memory or check your system.${colors[none]}\n\t${colors[yellow]}(${colors[none]}$HugePages_Free ${colors[yellow]}pages free out of ${colors[none]}$pagesRequired${colors[yellow]} requested. With that the highest value achieved right now is:${colors[none]} -mem $((($HugePages_Free * HugepagesizeMB)))${colors[yellow]})${colors[none]}\n" + printer "${colors[red]}\t\tMountpoint ${colors[none]}${hugePath}${colors[red]} not found in ${colors[none]}/proc/mounts${colors[red]} as type hugetlbfs?" + + if [ "$hugePathDefault" -eq 1 ] + then + printer "\t\t${colors[red]}Should typically be mounted by default in Linux. Please try to mount $hugePath" + printer "\t\t${colors[red]}manually or if you've configured a custom mountpoint please specify it." + else + printer "${colors[red]}\t\tIf you don't have a custom hugepage mount such as for" + printer "${colors[red]}\t\t1GB hugepage preallocation at boot then just use ${colors[none]}-hugepages${colors[red]}" + printer "${colors[red]}\t\twithout an argument to use the default system hugepages." + fi + printer "${colors[red]}\t\tPanicking." exit 1 - else - permissionsManager takeown /dev/hugepages - echo "Hugepages ready." fi -elif isDry + hugepageSize="$(grep $hugePath /proc/mounts | grep -Po '(?<=pagesize=).+[A-Z]')" + if [[ "$hugepageSize" =~ 'M' ]] then - echo -e "${colors[green]}-hugepages\tacknowledged [DRY]${colors[none]}" -fi + hugepageSizeMB="$(grep -Eo '[0-9]+' <<< $hugepageSize)" + elif [[ "$hugepageSize" =~ 'G' ]] + then + hugepageSizeMB="$(grep -Eo '[0-9]+' * 1024 ))) <<< $hugepageSize" + else + printer "${color[red]}\t\tSomething went wrong, I was unable to determine ${hugePath}'s hugepage size via the mountpoint. Panicking." ; exit 1 + fi + hugepageSizeKB="$((($hugepageSizeMB * 1024)))" + printer "${colors[green]}\t\tThe selected hugepage mount's size per page is ${hugepageSizeMB}M" -if [ "$HUGEPAGES" == "1" ] && [ -z "$hugeMount" ] && ! isDry -then - echo -e "${colors[red]}Unable to find the hugepage mount at $hugePath. Hugepages is not enabled for this run.${colors[none]}" + pagesRequired="$((($guestmemoryMB / $hugepageSizeMB)))" + printer "${colors[green]}\t\t${pagesRequired} pages will be deployed for ${guestmemoryMB}MB of VM memory." + if ! isDry + then + if [ $(readHugeInfo ${hugepageSizeKB} free_hugepages) -ge $pagesRequired ] + then + printer "\t\t${colors[green]}Hugepages already preallocated with enough free for guest, using those." + else + echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null && printer "\t\tMemory cache dropped for hugepages..." + echo -ne "\t\tRunning compact_memory... " ; echo 1 | sudo tee /proc/sys/vm/compact_memory >/dev/null && printer "${colors[green]}OK." + echo "$pagesRequired" | writeHugeInfo ${hugepageSizeKB} nr_hugepages ; hugepagesSelfAllocated=1 + printer "\t\tGiving hugepages a moment to allocate." + while [ $(readHugeInfo ${hugepageSizeKB} nr_hugepages) -lt $pagesRequired ] + do + echo -ne '.' + count=$((($count+1))) ; if [ $count -gt 30 ]; then break; fi + sleep 1 + done ; echo '' # Newline + hugepagesFree="$(readHugeInfo ${hugepageSizeKB} free_hugepages)" + if [[ "$(readHugeInfo ${hugepageSizeKB} free_hugepages)" -lt "$pagesRequired" ]] + then + printer "${colors[red]}Error, couldn't allocate all required pages. Please free up memory or check your system." + printer "\t${colors[yellow]}(${colors[none]}${hugepagesFree} ${colors[yellow]}pages free out of ${pagesRequired}${colors[yellow]} requested. With that the highest value achieved right now is: -mem $((($hugepagesFree * hugepageSizeMB)))${colors[yellow]})\n" + exit 1 + fi + permissionsManager takeown $hugePath + printer "\n\n${color[green]}Hugepages ready." + hugeArgs="-mem-path $hugePath" + fi + fi fi # Make the bridge @@ -621,11 +667,11 @@ if [[ ! -z "$bridgeArgs" ]]; then bridger start $bridgeArgs ; fi # Put core QEMU arguments together machineArgs="-machine q35,accel=kvm,kernel_irqchip=on" # kernel_irqchip helps prevent PCI Error Code 34 on GPUs. - cpuArgs="-cpu host,kvm=off" + if [ ! -z "$hyperv" ] then - echo -e "${colors[green]}-hyperv\t\tspecified, HyperV Enlightenments will be enabled for this run.${colors[none]}" + printer "${colors[green]}-hyperv\t\tspecified, HyperV Enlightenments will be enabled for this run." cpuArgs="${cpuArgs},hv-frequencies,hv-relaxed,hv-reset,hv-runtime,hv-spinlocks=0x1fff,hv-stimer,hv-synic,hv-time,hv-vapic,hv-vpindex" fi @@ -635,39 +681,49 @@ if [[ ! -z "$bridgeArgs" ]]; then bridger start $bridgeArgs ; fi if [[ ! -z "${!disks[@]}" ]] then - echo -e "${colors[green]}-image(s)\tspecified, using disks this run.${colors[none]}" + printer "${colors[green]}-image(s)\tspecified, using disks this run." else - echo -e "${colors[yellow]}-image\t\tnot specified. This VM will run diskless.${colors[none]}" + printer "${colors[yellow]}-image\t\tnot specified, This VM will run diskless." fi for disk in ${!disks[@]} do ioCounter=$((($ioCounter+1))) if [ ! -z "${disks[$disk]}" ]; then diskFormat=",format=${disks[$disk]}"; else unset diskFormat ; fi - echo -e "\t\t${colors[none]}${ioCounter} ${colors[blue]}${disk}${diskFormat}${colors[none]}" + printer "\t\t${ioCounter} ${colors[blue]}${disk}${diskFormat}" permissionsManager takeown ${disk} coreArgs="$coreArgs -drive file=$disk,if=none,discard=on,id=drive${ioCounter}${diskFormat} -device virtio-blk-pci,drive=drive${ioCounter},id=virtio${ioCounter}" done if [ ! -z "$isos" ] then - echo -e "${colors[green]}-iso(s)\tattached${colors[none]}" + printer "${colors[green]}-iso(s)\tattached" for iso in ${isos[@]} do ioCounter=$((($ioCounter+1))) - echo -e "\t\t${ioCounter} ${colors[blue]}$iso${colors[none]}" + printer "\t\t${ioCounter} ${colors[blue]}$iso" coreArgs="$coreArgs -drive file=${iso},index=${ioCounter},media=cdrom" done fi + +if [ ! -z "$romfilePath" ] +then + printer "${colors[green]}-romfile\tspecified, if a GPU is detected in the -pci arguments this romfile will be used." + printer "\t\t$romfilePath" + if [ ! -f "$romfilePath" ] ;then printer "${colors[red]}\t\tBut the romfile doesn't appear to exist? Panicking" ; exit 1 ; fi + printer "\t\t${colors[yellow]}Please confirm your romfile is safe with a project such as rom-parser before using this feature" + sleep 3 +fi + if [[ ! -z "$usbREGEX" ]] && [[ ! -z "$pciREGEX" ]] || [[ -z "$DISPLAY" ]] ; then coreArgs="$coreArgs -nographic -vga none" ; fi # Create X11 window if no passthrough used. Also if $DISPLAY isn't set. if [ ! -z "$lookingglass" ] then - echo -e "${colors[green]}-lookingglass\tspecified, we'll start qemu with a spice server and shared memory for it.${colors[none]}" + printer "${colors[green]}-lookingglass\tspecified, we'll start qemu with a spice server and shared memory for it." # Bad way to do this but wanted to keep them readable individually - lgArgs="$lgArgs -object {'qom-type':'memory-backend-file','id':'shmmem-shmem0','mem-path':'/dev/shm/looking-glass','size':67108864,'share':true}" # Memory currently hardcoded to 64M, can be changed in this script if needed. Plan to take resolution as an argument in future. - lgArgs="$lgArgs -device ivshmem-plain,id=shmem0,memdev=shmmem-shmem0,id=lookingglass" + lgArgs="$lgArgs -object {'qom-type':'memory-backend-file','id':'lgMemory0','mem-path':'/dev/shm/looking-glass','size':67108864,'share':true}" # Memory currently hardcoded to 64M, can be changed in this script if needed. Plan to take resolution as an argument in future. + lgArgs="$lgArgs -device ivshmem-plain,id=shmem0,memdev=lgMemory0,id=lookingglass" lgArgs="$lgArgs -chardev spicevmc,id=charchannel0,name=vdagent" lgArgs="$lgArgs -device virtio-serial" lgArgs="$lgArgs -chardev socket,path=/dev/shm/vmsocket,server=on,wait=off,id=vmsocket" @@ -680,7 +736,7 @@ fi if [ ! -z "$extras" ] then - echo -e "${colors[green]}-extras\t\tspecified, we'll start qemu with these extra arguments:${colors[none]}\t$extras" + printer "${colors[green]}-extras\t\tspecified, we'll start qemu with these extra arguments:${colors[none]}\t$extras" fi echo '' # Section separation echo @@ -689,14 +745,14 @@ networkArgs="-device virtio-net,netdev=network0" if [[ ! -z "$bridgeArgs" ]]; then networkArgs="$networkArgs -netdev tap,id=network0,ifname=tap0,script=no,downscript=no" # Bridge - echo -e "${colors[green]}-bridge\t\tspecified, VM will be bridged to the host with a tap adapter.${colors[none]}" + printer "${colors[green]}-bridge\t\tspecified, VM will be bridged to the host with a tap adapter." elif [[ ! -z "$noNet" ]] then networkArgs="" - echo -e "${colors[green]}-nonet specified, there will be no virtual networking for the VM this run.${colors[none]}" + printer "${colors[green]}-nonet specified, there will be no virtual networking for the VM this run." else networkArgs="$networkArgs -netdev user,id=network0" # NAT - echo -e "${colors[yellow]}No network arguments (-bridge/-nonet) specified, using a NAT adapter for VM networking this run.${colors[none]}" + printer "${colors[yellow]}No network arguments (-bridge/-nonet) specified, using a NAT adapter for VM networking this run." fi # Remove any leading/trailing pipes from the regex just in case. @@ -717,12 +773,12 @@ if isDry; then echo "---------------------" echo "Here are the completed args from this DRY run:" - echo -e "Core:\n${colors[blue]} $coreArgs $hugeArgs${colors[none]}" - if [ ! -z "$networkArgs" ]; then echo -e "Net :\n${colors[blue]}${networkArgs}${colors[none]}" ; fi - if [ ! -z "$usbArgs" ] ; then echo -e "USB :\n${colors[blue]}${usbArgs}${colors[none]}" ; fi - if [ ! -z "$pciArgs" ] ; then echo -e "PCI :\n${colors[blue]}${pciArgs}${colors[none]}" ; fi - if [ ! -z "$lgArgs" ] ; then echo -e "Looking Glass:\n${colors[blue]}${lgArgs}${colors[none]}" ; fi - if [ ! -z "$extras" ] ; then echo -e "Extras:\n${colors[blue]}${extras}${colors[none]}" ; fi + printer "Core:\n${colors[blue]} $coreArgs $hugeArgs" + if [ ! -z "$networkArgs" ]; then printer "Net :\n${colors[blue]}${networkArgs}" ; fi + if [ ! -z "$usbArgs" ] ; then printer "USB :\n${colors[blue]}${usbArgs}" ; fi + if [ ! -z "$pciArgs" ] ; then printer "PCI :\n${colors[blue]}${pciArgs}" ; fi + if [ ! -z "$lgArgs" ] ; then printer "Looking Glass:\n${colors[blue]}${lgArgs}" ; fi + if [ ! -z "$extras" ] ; then printer "Extras:\n${colors[blue]}${extras}" ; fi echo -ne "\nRun the script with the same arguments again and include ${colors[red]}-run${colors[none]} to actually pass these to qemu-system-x86_64\n" exit 0 else